From 149170522d1c0146292d393f453dbd1929cd5c17 Mon Sep 17 00:00:00 2001 From: 0x8f701 Date: Thu, 17 Sep 2026 10:32:23 +0800 Subject: [PATCH 1/3] Realm recovery: catch-up, bounded scans, and E2E runbook - Recover P2P votes and proposal bodies; stage with RAII and drop GC, freeze, and certificates - Bound leaf scans, batch validator and height reads, and rehash sparse paths - Separate replay from verification; detach faucet claims; add a shield-address example - Add the recovery E2E runbook and rollback operations; keep e2e-evidence gitignored --- .gitignore | 3 + AGENTS.md | 27 +- Makefile | 1 + README.md | 2 +- .../psy_user_cli/examples/derive_shield.rs | 25 + .../psy_prover/src/local/native/faucet.rs | 46 +- dev/locSetupPolicy.test.ts | 250 +- dev/locSetupPolicy.ts | 138 +- dev/locSetupV4.ts | 189 +- docs/src/dev/bridge-common-operations.md | 4 +- docs/src/dev/devnet-launcher-reference.md | 85 +- docs/src/dev/devnet_lifecycle.md | 12 +- docs/src/dev/processors.md | 2 + docs/src/dev/realm-recovery-e2e.md | 606 +++++ docs/src/dev/rollback-operations.md | 298 +++ docs/src/node/getting_started.md | 8 +- parth_common/src/tree_sync/traits.rs | 2 +- .../src/coordinator/standard_edge_rpc.rs | 2 +- psy_cli/psy_node_cli/src/node/realm_p2p.rs | 169 +- .../src/node/startup_plonky2_scylla.rs | 70 +- .../src/node/startup_processor_jtmb_scylla.rs | 74 +- .../bindings/PsyRealmUserUpdateQueueItem.ts | 2 +- psy_data/src/guta/realm_finalize.rs | 112 +- psy_data/src/p2p/limits.rs | 39 +- psy_data/src/p2p/messages.rs | 368 ++- psy_data/src/p2p/mod.rs | 8 +- psy_data/src/queue_items/realm_user_update.rs | 16 +- .../global_user_tree/db_loader_sub_root.rs | 318 ++- psy_node_common/src/coordinator/edge/api.rs | 3 +- .../src/coordinator/edge/handler.rs | 174 +- .../src/guta_planner/realm_guta_planner.rs | 69 +- .../realm_guta_planner_tests/test_env.rs | 53 +- psy_node_common/src/p2p/realm_coordinator.rs | 3 +- psy_node_common/src/queue/gatherer.rs | 122 +- psy_node_common/src/realm/edge/handler.rs | 4 +- .../src/realm/network/behaviour.rs | 38 +- psy_node_common/src/realm/network/codec.rs | 118 +- psy_node_common/src/realm/network/drive.rs | 685 ++++- psy_node_common/src/realm/network/mod.rs | 126 +- .../src/realm/network/reassembly.rs | 15 +- psy_node_common/src/realm/network/startup.rs | 4 +- .../src/realm/processor/catchup.rs | 388 +++ .../src/realm/processor/consensus.rs | 168 +- .../src/realm/processor/core/mod.rs | 16 +- .../src/realm/processor/core/process_block.rs | 291 +- .../src/realm/processor/core/runner.rs | 180 +- .../src/realm/processor/core/startup.rs | 177 +- psy_node_common/src/realm/processor/create.rs | 38 +- .../src/realm/processor/db/commit.rs | 369 +-- .../src/realm/processor/db/core.rs | 24 + .../src/realm/processor/db/genesis.rs | 359 +++ .../src/realm/processor/db/init.rs | 1145 +++++--- psy_node_common/src/realm/processor/db/mod.rs | 1 + .../src/realm/processor/db/sync.rs | 359 ++- .../gatherers/realm_end_cap_gatherer.rs | 30 + psy_node_common/src/realm/processor/mod.rs | 5 +- .../src/realm/processor/proposal_store.rs | 509 ++++ .../src/realm/processor/recovery.rs | 2384 +++++++++++++++++ psy_node_core/src/config/node_start_config.rs | 6 + .../src/p2p/traits/realm_coordinantor.rs | 2 +- psy_node_core/src/p2p/validator_lookup.rs | 298 ++- .../psy_core_db/core_implementation/base.rs | 2 +- .../core_implementation/constants.rs | 1 + psy_node_core/src/psy_core_db/traits/full.rs | 10 + .../src/psy_core_db/v3_implementation/full.rs | 60 +- psy_node_core/src/store/traits/core_db.rs | 8 + psy_node_scylla/src/core_db.rs | 15 + psy_node_scylla/src/tables/merkle/zero.rs | 6 +- psy_node_scylla/tests/zero_id_dump_v2.rs | 74 +- psy_node_store_memory/src/v2/cbs_store.rs | 35 + 70 files changed, 9512 insertions(+), 1738 deletions(-) create mode 100644 client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs create mode 100644 docs/src/dev/realm-recovery-e2e.md create mode 100644 docs/src/dev/rollback-operations.md create mode 100644 psy_node_common/src/realm/processor/catchup.rs create mode 100644 psy_node_common/src/realm/processor/db/genesis.rs create mode 100644 psy_node_common/src/realm/processor/proposal_store.rs create mode 100644 psy_node_common/src/realm/processor/recovery.rs diff --git a/.gitignore b/.gitignore index 282623ea..1c470df5 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ Thumbs.db dist/ local_checkpoints/ +e2e-evidence/ logs/ /logs_*/ /db @@ -75,6 +76,8 @@ AGENTS.md ISSUES.md TASKS.md MEMORY.md +PLAN.md +docs/src/audit/ # Generated ts-rs bindings. parth_core/bindings/ psy_core/bindings/ diff --git a/AGENTS.md b/AGENTS.md index 15706bf8..05add0d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -472,6 +472,8 @@ Violating any rule below requires an immediate fix before other work continues. 9. **Use structured web tooling.** Prefer repository readers or browser tooling over raw page dumps when those tools are available. 10. **Select the remote shell explicitly for SSH commands.** Never rely on the remote account's login shell to parse automation commands. Invoke the intended interpreter explicitly: use `ssh /usr/bin/bash -s` for Bash scripts or `ssh /usr/bin/bash -lc ''` for a single command. This prevents quoting, expansion, startup-file, and syntax differences in the remote environment from changing command behavior. 11. **Realm pipeline overlap is non-negotiable.** Candidate A proving, P2P consensus, and Coordinator inclusion must overlap with builder B accepting and speculatively aggregating real EndCaps on A's end root. Never replace this with a serial seal, inclusion wait, and resume barrier. Never keep B paused during A proving, consensus, or inclusion. A short seal and exact-root publication before A proving may seed B. Keep speculative intake separate from checkpoint-bound authoritative witness generation. Bind or rebuild B's authoritative graph only after a real checkpoint authenticates B's start root. Checkpoint proof guards remain fail-closed, and proof values must never be mutated. The current gatherer and processor contracts are `docs/src/dev/gatherers.md` and `docs/src/dev/processors.md`. +12. **No multiplicative cost. Linear is acceptable; a product of two dimensions is not.** Block production (`process_block`), Realm proposal publish and vote, per-checkpoint recovery and catch-up, and network request handlers may be linear in one dimension — one pass over N items, or one round trip per item — but never the product of two. Prohibited: scanning the whole member or peer set once per item (O(N×M)); iterating every chunk or slice of an item with work proportional to the whole item (O(N×payload), for example re-reading, re-decoding, or re-hashing a full object once per chunk); verifying the same artifact twice on one path; and any loop whose body cost grows with the number of items remaining. Choose one shape instead: one batched request per peer covering the whole window, or one request per item against a single chosen peer with failover bounded to a constant number of attempts per batch rather than per item. Total cost proportional to payload size is acceptable only when it is irreducible data volume paid once. When a call site cannot meet this bound, fix the boundary that forces the repetition instead of the caller. State the per-unit and total cost in the change description. + A peer-supplied candidate is a hint only — full per-item authentication still runs, and authentication across distinct checkpoints is not duplicate work. ## Core Engineering Principles @@ -492,6 +494,8 @@ Violating any rule below requires an immediate fix before other work continues. 15. **Fixes before features; features before improvements.** Any user-requested or verified in-scope fix has higher priority than starting or resuming new feature work. Finish its implementation, focused verification, required different-model review, and narrow commit before feature work continues. Defects outside the active scope are reported and deferred unless the user explicitly brings them into scope. 16. **Evidence-first bug fixing.** Every bug fix follows this order; skipping a stage is a rejected fix: (1) reproduce the user-visible failure on the real affected surface and capture the artifact (log, RPC, fixture, or runtime trace); (2) write a failing test that encodes that reproduction; (3) fix the defect at the responsible boundary with root-cause evidence (`:`); (4) obtain different-model review of the staged diff with zero unresolved findings; (5) re-reproduce on the original surface and confirm the failing test now passes. Hypotheses and source inspection guide investigation but are never root-cause proof. Three failed speculative fixes without a captured artifact require stopping and capturing evidence before writing another line. 17. **Exact scope and stopping rule.** Complete every cumulative user instruction and named acceptance criterion, including directly required correctness work, but do not add adjacent improvements, recursive cleanup, speculative hardening, or optional redesign. Once requested behavior has executable evidence and every requested verification, review, documentation, and required cleanup item is complete, stop. +18. **No stopping before the goal is fully met.** Work continues until every deliverable has current executable evidence. A phase boundary, sub-step, pending review, or infrastructure stall is never a reason to stop the turn or park the objective; unblocked reachable work continues in the same turn. Waiting on external input is allowed only when the information is unreachable by any tool, and the stop must state exactly what is missing and what was tried. +19. **Decision ownership.** The user makes only the most important decisions and design rulings (approval boundaries, external-authority changes, irreversible operations such as push/deploy/publish, and explicit scope changes). Everything else — implementation choices, bug-fix approaches, internal API shapes, repair strategies, sequencing within an agreed milestone — is decided by the working agent with stated reasoning. Never bounce a decision back to the user that the agent can resolve from repository evidence; when a rule genuinely requires user agreement (this file says so explicitly), state the choice made and the alternative rejected rather than presenting an options list. ## Multi-Agent Ownership @@ -594,7 +598,7 @@ Required fields per note: 7. Do not embed task identifiers, phase numbers, or step numbers in code names, file names, comments, or commit messages. 8. Version numbers belong to the runtime revision field and migration manifests, not to type names. If a migration window forces two schemas to coexist, name both by role and remove the older schema in the next storage-layout revision. 9. Lifecycle and authority labels (`legacy`, `old`, `deprecated`, `retired`, `canonical`, `official`, `v1`, `v2`) never name code, files, or documentation. Keep exactly one optimal implementation. A proxy-upgrade storage mirror is named by what it holds (for example `ImportedTokenFlowConfig`) and is removed in the next storage-layout revision. -10. Prefer one common verb plus one concrete object for function names. Use direct verbs such as `get`, `set`, `check`, `read`, `write`, `load`, `save`, `add`, `remove`, `create`, `update`, `apply`, and `build`; avoid multi-verb names, process jargon, and subjective lifecycle labels when one plain action states the behavior. +10. Prefer one common verb plus one concrete object for function names. Use direct verbs such as `get`, `set`, `check`, `read`, `write`, `load`, `save`, `add`, `remove`, `create`, `update`, `apply`, and `build`; avoid multi-verb names, process jargon, and subjective lifecycle labels when one plain action states the behavior. `persist` is banned: write durable data with `save`/`write`, read it back with `load`/`read`; do not introduce `persist*`, `store` as a verb synonym for those, or other interchangeable synonyms for the same operation. 11. Name quantities as `__`, with the unit last: `remaining_backup_bytes`, `max_proof_bytes`, `processed_user_count`. Avoid sentence fragments such as `backup_bytes_remaining` and implementation terms such as `allocation` when the value is a validated size or count. 12. Use common verbs consistently: `get` reads an existing value; `load` assembles a domain value from durable storage; `read` decodes a file or byte stream; `build` derives a value without persistence; `create` makes a new stored or runtime object; `set` replaces a whole value; `update` changes part of a value; `apply` executes a state transition; `validate` checks untrusted or serialized input; `ensure` enforces an internal invariant and returns an error; `check` returns a state classification or health result. Do not use these as interchangeable synonyms. 13. Order composite domain names from scope to object to representation: `checkpoint_tree_root`, `validator_tree_proof`, `user_leaf_hash`, `gathering_checkpoint_id`. State qualifiers precede the domain name (`current_`, `next_`, `last_committed_`, `remaining_`, `expected_`, `actual_`); collections use plural nouns and identifiers end in `_id` or `_ids`. @@ -638,7 +642,7 @@ Required fields per note: 10. Do not add dead code. Every new constant, function, type, or wire tag must have a production caller in the same change; constants reachable only from `#[cfg(test)]` are dead code and must not land in production modules. Before adding a public item, search the workspace for callers; if none exist, do not add it. 11. Keep test-only code out of core production modules. Do not place test-only helpers, fixtures, domain constants, or wrappers beside production code, and do not leave test-only compile units in a deployable source tree as a temporary convenience. A production symbol used only by tests is either deleted (with its tests) or moved into the test module / test-only tree; unused test scaffolding is deleted, not kept "for later". 12. Before merge, verify no new dead code: grep each newly exported symbol for callers outside `#[cfg(test)]` and outside the package's test-only tree. Zero production, deploy, or runtime callers rejects leaving the symbol in a deployable source set. -13. Mandatory protocol behavior must not be gated behind feature flags or optional toggles that leave the pipeline silently bypassed. If required behavior is claimed as implemented, the default code path must execute it end-to-end and fail closed when its mandatory inputs (identity, witness material, configuration) are missing. Silent skip is a defect: either wire the mandatory path and fail loudly on missing inputs, or do not claim the feature exists. +13. Mandatory protocol behavior must not be gated behind feature flags or optional toggles that leave the pipeline silently bypassed. If required behavior is claimed as implemented, the default code path must execute it end-to-end and fail closed when its mandatory inputs (identity, witness material, configuration) are missing. Silent skip is a defect: either wire the mandatory path and fail loudly on missing inputs, or do not claim the feature exists. No `skip_*`-style verification-escape parameters, conditional enum discriminants, match arms, or helper indirection may reintroduce a removed bypass under a different name; deleting a check requires deleting the parameter/flag and every caller carrying it, not renaming the decision point. 14. After implementing code and tests, do not run tests immediately. First perform a detailed audit: review the code for critical correctness/security issues, verify test cases are sufficient to defend the changed contract (including failure paths and boundaries), and identify gaps. If audit finds problems, fix them first. Only after audit passes begin running tests. An audit that finds nothing must still list every changed file and confirm each was inspected; an evidence-free clean audit is invalid. 15. All Rust tests MUST run with `--release`. Debug-mode compilation of Plonky2 circuit construction is impractically slow and can mask timing-dependent behavior. Use `cargo test --release` for every test invocation, including focused module filters. Debug-mode test runs are invalid evidence. 16. All code, comments, documentation, tests, commit messages, and log messages MUST be in English. No Chinese (or any other language) in source files, doc comments, test fixtures, or documentation under this repository; localized runbooks belong in the external memory repository, not in checked-in sources. @@ -706,6 +710,23 @@ A change is rejected until any applicable item is corrected: 9. When explaining code to users, default to concise pseudocode plus the core function and parameter names. Show real implementation excerpts or full data structures only when needed to resolve ambiguity, prove a claim, or enable a concrete action. 10. Keep ordinary technical replies centered on the decision, observable behavior, material risk, and verification. +## Agent Communication Style + +1. Lead with the conclusion. Omit filler and repetition. If a response exceeds ~300 lines without structure, it is too verbose. +2. Explain architecture and protocol flows with ASCII charts, not prose walls. Every multi-step flow gets a diagram. +3. Explain logic with pseudocode showing the core function and parameter names. Real implementation excerpts only when pseudocode cannot resolve ambiguity. +4. Clarify module boundaries with an ownership table: which module owns which datum, who reads, who writes. +5. Use tables for classifications (mechanism vs necessity, component vs verdict) instead of narrative paragraphs. + + +## Naming Quality Checklist + +Every new name (function, file, directory, type, concept) must pass three tests. Any "no" means rename before landing. + +1. **Does the name say WHAT it does or contains?** The name must name a concrete domain concept, not a generic category. `objects/` fails (anything could be an object); `proposals/` passes (the directory contains proposals). `put_complete` fails ("complete" is an adjective, not an object); `store_proposal` passes ("store" is the action, "proposal" is the object). +2. **Can someone who has never seen the code understand it from the name alone?** `lookup` fails (lookup what? by what key?); `lookup_by_checkpoint` passes (self-contained). `active view` fails ("view" is a presentation term, not a domain concept); `applied set` passes ("applied" = committed to DB, "set" = the collection). +3. **Is there a more specific word that would be equally short?** If the answer is yes, use the more specific word. Never keep a vague name because renaming feels disruptive — vague names compound into unreadable codebases. +The pattern for functions: **verb + concrete object**. The pattern for files/directories: **concrete noun**. If the object slot is filled by a generic word (`data`, `result`, `obj`, `items`, `view`, `complete`), replace it with the actual domain noun. The same ban covers generic catch-all nouns: `metadata`, `manifest`, `info`, `payload`, `context`, `detail(s)`, `entry`, `item(s)`, `blob`, `misc` — each must be replaced by what the value actually is (e.g. `PsyProvingJobMetadata` holds reward-tree layout and dependencies → name it by those fields, not "metadata"; a reset marker file is a `reset_marker`, not a "manifest"). `metadata`/`manifest` are allowed only where the word names a real external API or an established on-disk format owned outside this workspace; new code must name the concrete content. `isolated`/`isolation` are also banned as names: say what the check actually does — untrusted records replayed against the authenticated baseline before any durable write — e.g. `verify_state_updates_against_baseline`, `baseline_replay`; do not name new functions, types, fields, or error strings with `isolat*`. `submission` is banned as a new name: name the concrete act (`end_cap_upload`, `proof_delivery`, `vote_publication`) or reuse the established wire/type name it wraps; `Submit*`/`AlreadySubmitted` survive only as existing external API/wire names owned by current shipped interfaces. ## Git Commit Rules 1. Commit each independent task or milestone separately. @@ -791,8 +812,6 @@ Use these GitHub artifacts as the reference source. Do not substitute a machine- 14. While waiting on any stack readiness marker or E2E step, read the service logs (`logs/*.txt`, stdout AND stderr files) at every poll tick for WARN/ERROR/panic markers, and report any failure with file and excerpt before waiting further. Port liveness and `Process initialized successfully` lines are necessary but not sufficient: the prove-proxy and worker logs carry the real init evidence (circuit-build progress, `[CFLI:PSY_PROVE_PROXY_STARTED]`), and a process can keep logging while the service it depends on has died. 15. Polling loops that sleep between checks MUST read logs each tick, not only check ports or process existence; a loop that waits on ports alone misses stderr panics, OOM kills, and supervisor teardown messages until the whole wait times out. 16. No single sleep may exceed 30 seconds. Every wait MUST name its explicit stop condition (log marker, RPC response, port, file, process state) and poll with short intervals, re-checking the condition each tick. Never sleep a fixed 120s/300s "hoping" the condition appeared: on each tick, either the condition is met (proceed), a failure is detected (investigate with evidence), or the bounded budget expires (stop and report). Unbounded or unconditional sleeps are forbidden. -17. Unless the user explicitly names `make restart` (or a rollback-control command), do NOT use it — ever. The default flow for every code change and every verification is a fresh full-service launch: `make shutdown` (non-purge, retains chain data unless purged) + `make run-all` with freshly built binaries. A restart reuses the old process generation and hides launch-order failures; only a fresh full launch proves the stack. Treat any unprompted `make restart` as a violation of this rule. -18. The gatherer/processor pipeline is blocking by design: gatherer phases hand off to the processor, and block production awaits gatherer results. Do not weaken or bypass this pipeline when fixing intake bugs; preserve the handoff/blocking semantics (finalization replies, queue-key transitions, ACK ordering) in any change. ## Psy E2E Reference Registry diff --git a/Makefile b/Makefile index 8d7c9df4..c93a65cc 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ SEPOLIA_RPC_URL ?= https://ethereum-sepolia-rpc.publicnode.com PSY_SKIP_BRANCH_CHECK ?= 1 PSY_SKIP_KEYSTORE ?= 1 PSY_SKIP_BUILD ?= 1 +PURGE ?= 1 # PROVING_BACKEND := jtmb-poseidon-goldilocks .PHONY: all build clean test check check-all deploy-contracts register-users query-chain-info run-all rollback-db rollback-stop rollback-resume staging-server restart restart-all shutdown clean-db run-dummy-prover config_gen_v2 generate-genesis-data generate-groth16 regen-groth16-keystore regen-bridge-agg-keystore export-solidity-verifier export-solidity-verifier-deposit export-solidity-verifier-withdrawal diff --git a/README.md b/README.md index 5141257a..3fad6e44 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ make build PSY_SKIP_BRANCH_CHECK=1 PSY_SKIP_KEYSTORE=1 make run-all ``` -The launcher runs in the foreground and starts Realm P2P with generated local keys and a public runtime config. Stop the stack with `make shutdown`. Logs are written under `./logs`. +The launcher runs in the foreground and starts Realm P2P with generated local keys and a public runtime config. Stop the stack with `make shutdown`; this defaults to purge and deletes persisted chain state, so use `PURGE=0 make shutdown` to stop while preserving state. Logs are written under `./logs`. To run some example transactions first run: ```bash diff --git a/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs b/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs new file mode 100644 index 00000000..be7fcd86 --- /dev/null +++ b/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs @@ -0,0 +1,25 @@ +//! Derive a receiver shield address for private transfers. +//! +//! Usage: derive_shield +//! Prints the shield address in the canonical form accepted by +//! `psy_user_cli private-transfer --receiver` (QHashOut serde string round-trip). + +use plonky2::field::types::Field; +use psy_crypto::shield_address::derive_shield_address; +use serde_json::to_string; + +fn parse_arg(value: &str, name: &str) -> anyhow::Result { + value.parse::().map_err(|_| anyhow::anyhow!("invalid {name}: {value}")) +} + +fn main() -> anyhow::Result<()> { + let mut args = std::env::args().skip(1); + let user_id: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing user_id"))?, "user_id")?; + let random0: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing random0"))?, "random0")?; + let random1: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing random1"))?, "random1")?; + let shield = derive_shield_address(user_id, random0, random1); + // serde round-trip: from_str parses exactly this string back. + println!("{}", to_string(&shield)?.replace('"', "")); + let _ = plonky2::field::goldilocks_field::GoldilocksField::ZERO; + Ok(()) +} diff --git a/client_prover/psy_prover/src/local/native/faucet.rs b/client_prover/psy_prover/src/local/native/faucet.rs index be712761..cde7a288 100644 --- a/client_prover/psy_prover/src/local/native/faucet.rs +++ b/client_prover/psy_prover/src/local/native/faucet.rs @@ -96,8 +96,8 @@ struct PsyFaucetService { // Same-operator mutual exclusion is handled by `operator_locks` below. wallet_session: Arc, claim_records: DashMap<(u64, u64), PsyFaucetClaimRecord>, - recipient_locks: DashSet, - operator_locks: DashSet, + recipient_locks: Arc>, + operator_locks: Arc>, window_checkpoints: u64, turnstile_secret: Option, require_turnstile: bool, @@ -259,8 +259,8 @@ impl PsyFaucetService { operators, wallet_session: Arc::new(wallet_session), claim_records: DashMap::new(), - recipient_locks: DashSet::new(), - operator_locks: DashSet::new(), + recipient_locks: Arc::new(DashSet::new()), + operator_locks: Arc::new(DashSet::new()), window_checkpoints, turnstile_secret, require_turnstile, @@ -339,24 +339,44 @@ impl PsyFaucetService { } Ok(()) } +} + +struct DashSetEntryGuard(Arc>, T); + +impl Drop for DashSetEntryGuard { + fn drop(&mut self) { + self.0.remove(&self.1); + } +} +impl PsyFaucetService { // Turnstile-gated entry, used by the public web frontend and the hosted // wallet verification page. - async fn claim(&self, input: PsyFaucetClaimRequest) -> Result { + async fn claim(self: &Arc, input: PsyFaucetClaimRequest) -> Result { self.verify_turnstile(input.turnstile_token.as_deref(), input.turnstile_state.as_deref()) .await?; self.claim_for_recipient(input).await } - async fn claim_for_recipient(&self, input: PsyFaucetClaimRequest) -> Result { + async fn claim_for_recipient(self: &Arc, input: PsyFaucetClaimRequest) -> Result { let recipient_user_id = input.recipient_user_id; - if self.recipient_locks.insert(recipient_user_id) { - let result = self.claim_locked(input).await; - self.recipient_locks.remove(&recipient_user_id); - result - } else { - Err(rpc_error("faucet claim already in progress for this recipient")) + if !self.recipient_locks.insert(recipient_user_id) { + return Err(rpc_error("faucet claim already in progress for this recipient")); } + // The claim runs in a detached task that owns the lock guards: a client + // disconnect drops this RPC future but never the locks, so the + // recipient/operator mutex windows span the whole proving-and-submit + // work exactly once. + let (result_tx, result_rx) = tokio::sync::oneshot::channel(); + let worker = Arc::clone(self); + tokio::spawn(async move { + let _recipient_guard = DashSetEntryGuard(Arc::clone(&worker.recipient_locks), recipient_user_id); + let result = worker.claim_locked(input).await; + let _ = result_tx.send(result); + }); + result_rx + .await + .map_err(|_| rpc_error("faucet claim task terminated"))? } async fn claim_locked(&self, input: PsyFaucetClaimRequest) -> Result { @@ -400,9 +420,9 @@ impl PsyFaucetService { continue; } tried_operator = true; + let _operator_guard = DashSetEntryGuard(Arc::clone(&self.operator_locks), operator.user_id); let submit_result = self.submit_with_operator(operator, input.recipient_user_id, amount).await; - self.operator_locks.remove(&operator.user_id); match submit_result { Ok(tx_hash) => { diff --git a/dev/locSetupPolicy.test.ts b/dev/locSetupPolicy.test.ts index e4cfcf56..781abb64 100644 --- a/dev/locSetupPolicy.test.ts +++ b/dev/locSetupPolicy.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from "bun:test"; import { applyEnvioCpuSetToCompose, + classifySupervisedExit, DEFAULT_PROVING_RAYON_THREADS, DEFAULT_WORKER_BATCH_SIZE, - FATAL_PROCESSOR_ERROR_MARKERS, + emptyRestartStreak, FAUCET_ENV_KEYS, + failureSignature, findCpuSetOverlap, formatBridgeRelayerKeystoreDecryptError, formatCpuSet, @@ -12,14 +14,20 @@ import { COORDINATOR_PROCESSOR_READY_MARKER, REALM_PROCESSOR_READY_MARKER, hasZstdMagic, - isFatalProcessorErrorLine, isCompilerFingerprintSource, isExactProcessorReadyLine, isLikelyWrongKeystorePassword, isTransientScyllaSchemaFailure, + MAX_IDENTICAL_FAILURE_RESTARTS, + nextRestartDelayMs, + nextSpawnRetryDelayMs, + shouldScheduleSupervisedRestart, parseCpuSet, parseEnvAssignments, + parseFatalProcessorErrorMarker, parseLscpuTopology, + parseRealmProcessorFailureLine, + planSupervisedRestart, PSY_DAPP_NESTED_PAYLOADS, PSY_DAPP_NESTED_SUBMODULES, PSY_SDK_GENESIS_CONFIG_REL, @@ -35,9 +43,9 @@ import { resolveWalletPasswordPolicy, selectNonEmptyEnv, resolveRealmWorkerCount, - shouldFatalRestartProcessor, shouldSkipBranchSync, } from "./locSetupPolicy"; +import type { RestartPlan, RestartStreak, SupervisedFailureCause } from "./locSetupPolicy"; describe("resolveRealmWorkerCount", () => { it("starts two realm workers for a full devnet by default", () => { @@ -248,44 +256,226 @@ describe("shouldSkipBranchSync", () => { }); }); -describe("isFatalProcessorErrorLine", () => { - it("detects the realm processor fatal marker", () => { - expect(isFatalProcessorErrorLine("[CFLI:PSY_REALM_PROCESSOR_ERROR] coordinator halted")).toBe(true); - }); - - it("detects the coordinator processor fatal marker", () => { - expect(isFatalProcessorErrorLine("2026-07-29 [CFLI:PSY_COORDINATOR_PROCESSOR_ERROR] boom")).toBe(true); +describe("parseFatalProcessorErrorMarker", () => { + it("parses exact CFLI tokens and ignores other lines", () => { + expect(parseFatalProcessorErrorMarker("[CFLI:PSY_REALM_PROCESSOR_ERROR] coordinator halted")).toBe("PSY_REALM_PROCESSOR_ERROR"); + expect(parseFatalProcessorErrorMarker("2026-07-29 [CFLI:PSY_COORDINATOR_PROCESSOR_ERROR] boom")).toBe("PSY_COORDINATOR_PROCESSOR_ERROR"); + expect(parseFatalProcessorErrorMarker("[CFLI:PSY_REALM_PROCESSOR_STARTED] up")).toBeNull(); + expect(parseFatalProcessorErrorMarker("realm_processor_failure realm_id=0 realm_sub_id=1 error=x")).toBeNull(); }); +}); - it("ignores ordinary processor log lines", () => { - expect(isFatalProcessorErrorLine("[CFLI:PSY_REALM_PROCESSOR_STARTED] up")).toBe(false); - expect(isFatalProcessorErrorLine("[REALM_CREATE] processor new done")).toBe(false); - expect(isFatalProcessorErrorLine("")).toBe(false); +describe("parseRealmProcessorFailureLine", () => { + it("captures the structured stderr event and ignores CFLI", () => { + const line = "realm_processor_failure realm_id=0 realm_sub_id=1 error=leaf-mismatch"; + expect(parseRealmProcessorFailureLine(line)).toBe(line); + expect(parseRealmProcessorFailureLine("[CFLI:PSY_REALM_PROCESSOR_ERROR] x")).toBeNull(); + expect(parseRealmProcessorFailureLine("Error: channel closed")).toBeNull(); + expect(parseRealmProcessorFailureLine("realm_processor_failureevil")).toBeNull(); + expect(parseRealmProcessorFailureLine("\u001b[0m\u001b[31m" + line + "\u001b[0m")).toBe(line); }); +}); - it("matches every advertised marker", () => { - for (const marker of FATAL_PROCESSOR_ERROR_MARKERS) { - expect(isFatalProcessorErrorLine(`prefix ${marker} suffix`)).toBe(true); +describe("supervised retry policy", () => { + const fatal: SupervisedFailureCause = { kind: "fatal-processor-error", marker: "PSY_REALM_PROCESSOR_ERROR" }; + const firstCause = "realm_processor_failure realm_id=0 realm_sub_id=1 error=leaf-mismatch"; + const firstIso = "2026-01-01T00:00:00.000Z"; + + function plan(overrides: { + restartCount?: number; + streak?: RestartStreak; + cause?: SupervisedFailureCause; + firstCause?: string | null; + healthySinceMs?: number; + nowMs?: number; + observedAtIso?: string; + } = {}): RestartPlan { + return planSupervisedRestart({ + restartCount: 0, + streak: emptyRestartStreak(), + cause: fatal, + firstCause, + healthySinceMs: 0, + nowMs: 1_000, + observedAtIso: firstIso, + ...overrides, + }); + } + + function repeat(count: number, cause: SupervisedFailureCause = fatal): RestartPlan { + let streak = emptyRestartStreak(); + let restartCount = 0; + let last = plan({ streak, restartCount, cause }); + for (let i = 0; i < count; i += 1) { + last = plan({ streak, restartCount, cause, observedAtIso: `2026-01-01T00:00:0${i}.000Z` }); + streak = last.streak; + restartCount = last.restartCount; } + return last; + } + + it("keeps the existing 1s exponential backoff capped at 30s", () => { + expect([1, 2, 3, 4, 5, 6, 7].map(nextRestartDelayMs)).toEqual([1000, 2000, 4000, 8000, 16000, 30000, 30000]); + expect(nextSpawnRetryDelayMs(30000)).toBe(60000); + }); + + it("classifies from typed marker, signal, and exit code", () => { + expect(classifySupervisedExit({ + fatalProcessorErrorMarker: "PSY_REALM_PROCESSOR_ERROR", + dependencyRestartRequested: false, + signalCode: "SIGTERM", + exitCode: 143, + })).toEqual({ kind: "fatal-processor-error", marker: "PSY_REALM_PROCESSOR_ERROR" }); + expect(classifySupervisedExit({ + fatalProcessorErrorMarker: null, + dependencyRestartRequested: false, + signalCode: "SIGTERM", + exitCode: null, + })).toEqual({ kind: "signaled", signal: "SIGTERM" }); + expect(classifySupervisedExit({ + fatalProcessorErrorMarker: null, + dependencyRestartRequested: false, + signalCode: null, + exitCode: 1, + })).toEqual({ kind: "exited", code: 1 }); + }); + + it("restarts the first identical typed failure on the 1s backoff", () => { + const first = plan(); + expect(first.action).toBe("restart"); + expect(first.restartCount).toBe(1); + expect(first.delayMs).toBe(1000); + expect(first.streak.identicalRepeats).toBe(1); + expect(first.streak.firstCause).toBe(firstCause); + expect(failureSignature(first.streak.cause!)).toBe(failureSignature(fatal)); + }); + + it("stops automatic restarts on the fifth identical typed failure", () => { + expect(MAX_IDENTICAL_FAILURE_RESTARTS).toBe(4); + const last = repeat(MAX_IDENTICAL_FAILURE_RESTARTS + 1); + expect(last.action).toBe("limit"); + expect(last.delayMs).toBe(0); + expect(last.restartCount).toBe(4); + expect(last.streak.identicalRepeats).toBe(5); + expect(last.streak.firstCause).toBe(firstCause); + expect(last.streak.firstObservedAtIso).toBe("2026-01-01T00:00:00.000Z"); + }); + + it("still restarts before the identical-failure cap", () => { + const last = repeat(MAX_IDENTICAL_FAILURE_RESTARTS); + expect(last.action).toBe("restart"); + expect(last.restartCount).toBe(4); + expect(last.streak.identicalRepeats).toBe(4); + }); + + it("resets identical repeats when the typed cause changes but keeps the first causal stderr", () => { + const afterFatal = repeat(2); + const afterExit = plan({ + streak: afterFatal.streak, + restartCount: afterFatal.restartCount, + cause: { kind: "exited", code: 1 }, + firstCause: "realm_processor_failure realm_id=0 realm_sub_id=1 error=other", + }); + expect(afterExit.action).toBe("restart"); + expect(afterExit.streak.identicalRepeats).toBe(1); + expect(afterExit.streak.cause).toEqual({ kind: "exited", code: 1 }); + expect(afterExit.streak.firstCause).toBe(firstCause); + expect(afterExit.streak.firstObservedAtIso).toBe("2026-01-01T00:00:00.000Z"); + }); + + it("keeps the first causal stderr for the life of the streak", () => { + const first = plan({ observedAtIso: firstIso }); + const later = plan({ + streak: first.streak, + restartCount: first.restartCount, + firstCause: "realm_processor_failure realm_id=0 realm_sub_id=1 error=later-symptom", + observedAtIso: "2026-01-01T00:00:10.000Z", + }); + expect(later.streak.firstCause).toBe(firstCause); + expect(later.streak.firstObservedAtIso).toBe(firstIso); + expect(later.streak.identicalRepeats).toBe(2); }); -}); -describe("shouldFatalRestartProcessor", () => { - it("requests a restart the first time a fatal marker appears", () => { - expect(shouldFatalRestartProcessor("[CFLI:PSY_REALM_PROCESSOR_ERROR] x", false)).toBe(true); - expect(shouldFatalRestartProcessor("[CFLI:PSY_COORDINATOR_PROCESSOR_ERROR] x", false)).toBe(true); + it("does not treat a long unready launch as a stable run", () => { + const longUnready = plan({ + restartCount: 4, + healthySinceMs: 0, + nowMs: 120_000, + }); + expect(longUnready.action).toBe("restart"); + expect(longUnready.restartCount).toBe(5); + expect(longUnready.delayMs).toBe(16_000); + expect(longUnready.streak.identicalRepeats).toBe(1); }); - it("does not request a restart for non-fatal lines", () => { - expect(shouldFatalRestartProcessor("[REALM_CREATE] processor new done", false)).toBe(false); - expect(shouldFatalRestartProcessor("ordinary stdout", false)).toBe(false); + it("resets backoff and streak after a ready run of at least 60s", () => { + const healthy = plan({ + restartCount: 4, + healthySinceMs: 1, + nowMs: 60_001, + }); + expect(healthy.action).toBe("restart"); + expect(healthy.restartCount).toBe(1); + expect(healthy.delayMs).toBe(1000); + expect(healthy.streak.identicalRepeats).toBe(1); + expect(healthy.streak.firstCause).toBe(firstCause); + }); + + it("uses the current spawn cause after a healthy reset, not the previous epoch", () => { + const causeA = "realm_processor_failure realm_id=0 realm_sub_id=1 error=A"; + const causeB = "realm_processor_failure realm_id=0 realm_sub_id=1 error=B"; + const epochA = plan({ firstCause: causeA, observedAtIso: firstIso }); + const afterHealthy = plan({ + streak: epochA.streak, + restartCount: epochA.restartCount, + firstCause: causeB, + healthySinceMs: 1, + nowMs: 60_001, + observedAtIso: "2026-01-01T00:02:00.000Z", + }); + expect(afterHealthy.action).toBe("restart"); + expect(afterHealthy.restartCount).toBe(1); + expect(afterHealthy.streak.identicalRepeats).toBe(1); + expect(afterHealthy.streak.firstCause).toBe(causeB); + expect(afterHealthy.streak.firstObservedAtIso).toBe("2026-01-01T00:02:00.000Z"); + }); + + it("does not treat later spawn-failure waits as a second healthy reset", () => { + const spawnFailed: SupervisedFailureCause = { kind: "spawn-failed" }; + const afterHealthyChild = plan({ + restartCount: 0, + healthySinceMs: 1, + nowMs: 60_001, + cause: spawnFailed, + firstCause: null, + }); + expect(afterHealthyChild.action).toBe("restart"); + expect(afterHealthyChild.restartCount).toBe(1); + let last = afterHealthyChild; + let nowMs = 60_001; + const failedSpawnMs = 20_000; + for (let i = 0; i < MAX_IDENTICAL_FAILURE_RESTARTS; i += 1) { + nowMs += failedSpawnMs + nextSpawnRetryDelayMs(last.delayMs); + last = plan({ + streak: last.streak, + restartCount: last.restartCount, + cause: spawnFailed, + firstCause: null, + healthySinceMs: 0, + nowMs, + }); + } + expect(nowMs - 60_001).toBeGreaterThan(60_000); + expect(last.action).toBe("limit"); + expect(last.streak.identicalRepeats).toBe(5); + expect(last.restartCount).toBe(4); }); - it("suppresses repeated kills once one has already been requested", () => { - const line = "[CFLI:PSY_REALM_PROCESSOR_ERROR] duplicated"; - expect(shouldFatalRestartProcessor(line, false)).toBe(true); - expect(shouldFatalRestartProcessor(line, true)).toBe(false); - expect(shouldFatalRestartProcessor("[CFLI:PSY_COORDINATOR_PROCESSOR_ERROR] other", true)).toBe(false); + it("does not schedule a restart after intentional stop or teardown", () => { + expect(shouldScheduleSupervisedRestart(false, false)).toBe(true); + expect(shouldScheduleSupervisedRestart(true, false)).toBe(false); + expect(shouldScheduleSupervisedRestart(false, true)).toBe(false); + expect(shouldScheduleSupervisedRestart(true, true)).toBe(false); }); }); diff --git a/dev/locSetupPolicy.ts b/dev/locSetupPolicy.ts index 6b982d2e..6aa4f180 100644 --- a/dev/locSetupPolicy.ts +++ b/dev/locSetupPolicy.ts @@ -350,26 +350,130 @@ export function isTransientScyllaSchemaFailure(errorText: string): boolean { }); } -// Log markers the processor binaries emit when they hit a fatal, unrecoverable -// error. Such a processor may keep running while producing empty blocks, so the -// devnet supervisor must terminate it and let auto-restart recreate it. -export const FATAL_PROCESSOR_ERROR_MARKERS: readonly string[] = [ - "[CFLI:PSY_REALM_PROCESSOR_ERROR]", - "[CFLI:PSY_COORDINATOR_PROCESSOR_ERROR]", -]; +// Processor binaries emit these CFLI tokens on a fatal, unrecoverable error. +export const FATAL_PROCESSOR_ERROR_MARKERS = [ + "PSY_REALM_PROCESSOR_ERROR", + "PSY_COORDINATOR_PROCESSOR_ERROR", +] as const; +export type FatalProcessorErrorMarker = (typeof FATAL_PROCESSOR_ERROR_MARKERS)[number]; + +/** Existing backoff: 1s exponential, cap 30s. Spawn-retry delay doubles, cap 60s. */ +export const RESTART_BACKOFF_CAP_MS = 30_000; +export const RESTART_SPAWN_RETRY_CAP_MS = 60_000; +/** Ready/healthy interval that resets backoff. Circuit-build time is not healthy. */ +export const RESTART_STABLE_RUN_MS = 60_000; +/** Consecutive identical typed failures that still auto-restart. The next identical failure stops. */ +export const MAX_IDENTICAL_FAILURE_RESTARTS = 4; +export const SUPERVISOR_RESTART_LIMIT_TAG = "[DevNet][supervisor][restart-limit]"; + +export type SupervisedFailureCause = + | { kind: "fatal-processor-error"; marker: FatalProcessorErrorMarker } + | { kind: "signaled"; signal: string } + | { kind: "exited"; code: number } + | { kind: "spawn-failed" } + | { kind: "dependency-restart" }; + +/** One retry record. First causal stderr is kept; kind is derived from cause. */ +export type RestartStreak = { + cause: SupervisedFailureCause | null; + firstCause: string | null; + firstObservedAtIso: string | null; + identicalRepeats: number; +}; + +export type RestartPlan = { + action: "restart" | "limit"; + restartCount: number; + delayMs: number; + streak: RestartStreak; +}; -/** True when a processor log line announces a fatal, unrecoverable error. */ -export function isFatalProcessorErrorLine(line: string): boolean { - return FATAL_PROCESSOR_ERROR_MARKERS.some((marker) => line.includes(marker)); +export function emptyRestartStreak(): RestartStreak { + return { cause: null, firstCause: null, firstObservedAtIso: null, identicalRepeats: 0 }; } -/** - * Decide whether a processor process emitting `line` should be terminated for - * a supervised restart. Returns false once a kill has already been requested - * (`alreadyRequested`), so duplicate log lines do not trigger repeated kills. - */ -export function shouldFatalRestartProcessor(line: string, alreadyRequested: boolean): boolean { - return !alreadyRequested && isFatalProcessorErrorLine(line); +/** Recheck after every backoff/spawn-retry sleep. Never clear intentionalStop to force a retry. */ +export function shouldScheduleSupervisedRestart(stopping: boolean, intentionalStop: boolean): boolean { + return !stopping && !intentionalStop; +} + +/** Single prose boundary: exact `[CFLI:]` → closed marker. */ +export function parseFatalProcessorErrorMarker(line: string): FatalProcessorErrorMarker | null { + for (const marker of FATAL_PROCESSOR_ERROR_MARKERS) { + if (line.includes(`[CFLI:${marker}]`)) return marker; + } + return null; +} + +/** Structured stderr event; never treat CFLI as the causal error. */ +export function parseRealmProcessorFailureLine(line: string): string | null { + const normalized = line.replace(/\u001b\[[0-9;]*m/g, "").trim(); + return normalized.startsWith("realm_processor_failure ") ? normalized : null; +} + +export function classifySupervisedExit(opts: { + spawnFailed?: boolean; + fatalProcessorErrorMarker: FatalProcessorErrorMarker | null; + dependencyRestartRequested: boolean; + signalCode: string | null; + exitCode: number | null; +}): SupervisedFailureCause { + if (opts.spawnFailed) return { kind: "spawn-failed" }; + if (opts.fatalProcessorErrorMarker !== null) { + return { kind: "fatal-processor-error", marker: opts.fatalProcessorErrorMarker }; + } + if (opts.dependencyRestartRequested) return { kind: "dependency-restart" }; + if (opts.signalCode) return { kind: "signaled", signal: opts.signalCode }; + return { kind: "exited", code: opts.exitCode ?? 0 }; +} + +export function failureSignature(cause: SupervisedFailureCause): string { + switch (cause.kind) { + case "fatal-processor-error": return `fatal-processor-error:${cause.marker}`; + case "signaled": return `signaled:${cause.signal}`; + case "exited": return `exited:${cause.code}`; + default: return cause.kind; + } +} + +export function nextRestartDelayMs(attempt: number): number { + return Math.min(RESTART_BACKOFF_CAP_MS, 1_000 * Math.pow(2, Math.min(attempt - 1, 5))); +} + +export function nextSpawnRetryDelayMs(restartDelayMs: number): number { + return Math.min(RESTART_SPAWN_RETRY_CAP_MS, restartDelayMs * 2); +} + +export function planSupervisedRestart(opts: { + restartCount: number; + streak: RestartStreak; + cause: SupervisedFailureCause; + firstCause: string | null; + healthySinceMs: number; + nowMs: number; + observedAtIso: string; +}): RestartPlan { + const stable = opts.healthySinceMs > 0 && opts.nowMs - opts.healthySinceMs >= RESTART_STABLE_RUN_MS; + const restartCount = stable ? 0 : opts.restartCount; + const streak = stable ? emptyRestartStreak() : opts.streak; + const firstCause = stable ? opts.firstCause : (streak.firstCause ?? opts.firstCause); + if (opts.cause.kind === "dependency-restart") { + return { action: "restart", restartCount: restartCount + 1, delayMs: nextRestartDelayMs(restartCount + 1), streak }; + } + const signature = failureSignature(opts.cause); + const same = streak.cause !== null && failureSignature(streak.cause) === signature; + const nextStreak: RestartStreak = same + ? { ...streak, firstCause: streak.firstCause ?? firstCause, identicalRepeats: streak.identicalRepeats + 1 } + : { + cause: opts.cause, + firstCause, + firstObservedAtIso: streak.firstObservedAtIso ?? opts.observedAtIso, + identicalRepeats: 1, + }; + if (nextStreak.identicalRepeats > MAX_IDENTICAL_FAILURE_RESTARTS) { + return { action: "limit", restartCount, delayMs: 0, streak: nextStreak }; + } + return { action: "restart", restartCount: restartCount + 1, delayMs: nextRestartDelayMs(restartCount + 1), streak: nextStreak }; } /** diff --git a/dev/locSetupV4.ts b/dev/locSetupV4.ts index 4728d14d..3eb77b85 100644 --- a/dev/locSetupV4.ts +++ b/dev/locSetupV4.ts @@ -17,7 +17,11 @@ import { PSY_SDK_GENESIS_CONFIG_REL, PSY_SDK_GENESIS_SUBMODULE, REALM_PROCESSOR_READY_MARKER, + SUPERVISOR_RESTART_LIMIT_TAG, applyEnvioCpuSetToCompose, + classifySupervisedExit, + emptyRestartStreak, + failureSignature, formatBridgeRelayerKeystoreDecryptError, formatPsyDappNestedSubmoduleRemedy, hasFaucetOperatorConfig, @@ -25,7 +29,12 @@ import { parseCpuSet, isExactProcessorReadyLine, isTransientScyllaSchemaFailure, + nextSpawnRetryDelayMs, + parseFatalProcessorErrorMarker, + parseRealmProcessorFailureLine, parseLscpuTopology, + planSupervisedRestart, + shouldScheduleSupervisedRestart, resolveCpuPartition, parseEnvAssignments, resolveCpuPartitionForAffinity, @@ -33,7 +42,6 @@ import { resolveRayonThreadCount, resolveRealmWorkerCount, resolveWalletPasswordPolicy, - shouldFatalRestartProcessor, resolveScyllaMemory, selectNonEmptyEnv, shouldSkipBranchSync, @@ -42,7 +50,12 @@ import { isCompilerFingerprintSource, s3CurlArgs, } from "./locSetupPolicy"; -import type { PsyDappNestedInitPlan, PsyDappNestedSubmodule } from "./locSetupPolicy"; +import type { + FatalProcessorErrorMarker, + PsyDappNestedInitPlan, + PsyDappNestedSubmodule, + RestartStreak, +} from "./locSetupPolicy"; /** * Retry processor creation only for the known transient Scylla schema family. @@ -560,7 +573,7 @@ export class RunningProcess { stdOutVisitor: ProcessLineVisitor = () => { }; stdErrVisitor: ProcessLineVisitor = () => { }; allOutputVisitor: ProcessLineVisitor = () => { }; - onExit: (code: number | null, signal: number | null) => void = () => { }; + onExit: (code: number | null, signal: string | null) => void = () => { }; /** Stable service name for supervisor logs (e.g. prove_proxy_0, bridge_relayer). */ name: string = ''; @@ -582,14 +595,19 @@ export class RunningProcess { initMaxRetries: number = 3; initRetryDelayMs: number = 2000; restartCount: number = 0; + startedAtMs: number = 0; + /** 0 until ready. Circuit build must not count as a healthy run. */ + healthySinceMs: number = 0; + restartStreak: RestartStreak = emptyRestartStreak(); hasExited: boolean = false; exitCode: number | null = null; - exitSignal: number | null = null; + exitSignal: string | null = null; supervisorObservedExit: boolean = false; intentionalStop: boolean = false; dependencyRestartRequested: boolean = false; - /** Fatal processor error already observed and signaled for supervised restart. */ - fatalRestartRequested: boolean = false; + fatalProcessorErrorMarker: FatalProcessorErrorMarker | null = null; + firstCausalStderr: string | null = null; + fatalWitnessWrite: Promise | null = null; constructor(proc: Bun.Subprocess, stdOutVisitor?: ProcessLineVisitor, stdErrVisitor?: ProcessLineVisitor, allOutputVisitor?: ProcessLineVisitor) { this.proc = proc; @@ -618,6 +636,10 @@ export class RunningProcess { let lines = this.lineBufferStdErr.split('\n'); this.lineBufferStdErr = lines.pop() || ''; lines.forEach(line => { + if (!this.firstCausalStderr) { + const causal = parseRealmProcessorFailureLine(line); + if (causal) this.firstCausalStderr = causal; + } this.stdErrVisitor(line, this); this.allOutputVisitor(line, this); }); @@ -690,6 +712,8 @@ export class RunningProcess { const runningProcess = new RunningProcess(proc, options.stdOutVisitor, options.stdErrVisitor, options.allOutputVisitor); runningProcess.cmds = cmds.slice(); + runningProcess.startedAtMs = Date.now(); + runningProcess.healthySinceMs = runningProcess.startedAtMs; runningProcess.spawnOptions = { cwd: options.cwd, stdOutVisitor: options.stdOutVisitor, @@ -701,7 +725,6 @@ export class RunningProcess { env: options.env, appendLogs: options.appendLogs, }; - const outputPumps: Promise[] = []; const pumpOutput = async ( readableStream: AsyncIterable, @@ -771,10 +794,11 @@ export class RunningProcess { (async () => { const code = await proc.exited; await Promise.allSettled(outputPumps); + const exitSignal = proc.signalCode ?? null; runningProcess.exitCode = code; - runningProcess.exitSignal = null; + runningProcess.exitSignal = exitSignal; runningProcess.hasExited = true; - runningProcess.onExit(code, null); + runningProcess.onExit(code, exitSignal); })(); return runningProcess; @@ -803,6 +827,7 @@ export class RunningProcess { initialized = true; clearTimeout(timeout); process.initializationReady = true; + process.healthySinceMs = Date.now(); resolve(process); }; const allOutputVisitor: ProcessLineVisitor = (line: string, process: RunningProcess) => { @@ -825,9 +850,10 @@ export class RunningProcess { // Keep the ORIGINAL visitor for supervisor restarts (not the init-hint wrapper). proc.spawnOptions.allOutputVisitor = options.allOutputVisitor; proc.hintDetector = hintDetector; + if (!proc.initializationReady) proc.healthySinceMs = 0; proc.useInitHint = true; const prevOnExit = proc.onExit.bind(proc); - proc.onExit = (code: number | null, signal: number | null) => { + proc.onExit = (code: number | null, signal: string | null) => { if (!settled) { settled = true; clearTimeout(timeout); @@ -3441,6 +3467,8 @@ async function killKnownProcesses(): Promise { "psy-dapp/apps/bridge", "psy-dapp/apps/ide", "psy-dapp/apps/explorer", + "psy-dapp/mode-a-web-wallet-bridge", + "vite.js --host 0.0.0.0 --port 5179", "pnpm dev", "envio/bin.js", "envio-linux", @@ -3456,7 +3484,7 @@ async function killKnownProcesses(): Promise { } async function killKnownPorts(): Promise { - const ports: number[] = [3000, 5433, 8080, 8081, 8545, 9545, 10545, 9898, 9998, 5174, 5175, 5176, 5177, 5178]; + const ports: number[] = [3000, 5433, 8080, 8081, 8545, 9545, 10545, 9898, 9998, 5174, 5175, 5176, 5177, 5178, 5179]; for (let p = 1337; p <= 1346; p++) ports.push(p); for (let p = 9999; p <= 10008; p++) ports.push(p); for (let p = 13380; p <= 14679; p++) ports.push(p); @@ -3742,26 +3770,43 @@ class DevNetProcessManager { if (!this.autoRestartEnabled() || !this.isProcessorProcess(p)) return; const originalVisitor = p.allOutputVisitor; p.allOutputVisitor = (line: string, process: RunningProcess) => { - if (shouldFatalRestartProcessor(line, process.fatalRestartRequested)) { - process.fatalRestartRequested = true; - console.warn( - `[DevNet][supervisor] fatal processor error detected for '${process.name}' ` + - `(pid=${process.pid}); terminating for supervised restart` - ); - try { - process.signalProcessGroup("SIGTERM"); - } catch (err) { - console.warn(`[DevNet][supervisor] failed to signal fatal processor '${process.name}': ${err}`); - } + const marker = parseFatalProcessorErrorMarker(line); + if (marker !== null && process.fatalProcessorErrorMarker === null) { + process.fatalProcessorErrorMarker = marker; + process.fatalWitnessWrite = this.writeFatalWitnessThenTerminate(process, marker); } originalVisitor(line, process); }; } + private async writeFatalWitnessThenTerminate( + process: RunningProcess, + marker: FatalProcessorErrorMarker, + ): Promise { + const witness = + `[DevNet][supervisor][restart-failure] service=${process.name} ` + + `cause=${failureSignature({ kind: "fatal-processor-error", marker })} ` + + `marker=${marker} pid=${process.pid}`; + console.warn(witness); + try { + await RunningProcess.appendLogBanner(process.spawnOptions.stdoutLogFile, witness); + await RunningProcess.appendLogBanner(process.spawnOptions.stderrLogFile, witness); + } catch (err) { + console.warn(`[DevNet][supervisor] failed to write fatal witness for '${process.name}': ${err}`); + } + if (this.stopping || process.intentionalStop || process.hasExited) return; + try { + process.signalProcessGroup("SIGTERM"); + } catch (err) { + console.warn(`[DevNet][supervisor] failed to signal fatal processor '${process.name}': ${err}`); + } + } + private async handleSupervisedExit( previous: RunningProcess, code: number | null, - signal: number | null, + signal: string | null, + spawnFailed: boolean = false, ): Promise { const name = previous.name || "process"; if (this.stopping || previous.intentionalStop) { @@ -3772,28 +3817,74 @@ class DevNetProcessManager { console.warn(`[DevNet][supervisor] process '${name}' exited (code=${code}, signal=${signal}); auto-restart disabled via PSY_NO_AUTO_RESTART=1`); return; } + if (previous.fatalWitnessWrite) { + try { + await previous.fatalWitnessWrite; + } catch (err) { + console.warn(`[DevNet][supervisor] fatal witness write failed for '${name}': ${err}`); + } + } + + const nowMs = Date.now(); + const observedAtIso = new Date(nowMs).toISOString(); + const cause = classifySupervisedExit({ + spawnFailed, + fatalProcessorErrorMarker: previous.fatalProcessorErrorMarker, + dependencyRestartRequested: previous.dependencyRestartRequested, + signalCode: signal, + exitCode: code, + }); + const plan = planSupervisedRestart({ + restartCount: previous.restartCount, + streak: previous.restartStreak, + cause, + firstCause: previous.firstCausalStderr, + healthySinceMs: spawnFailed ? 0 : previous.healthySinceMs, + nowMs, + observedAtIso, + }); + previous.restartCount = plan.restartCount; + previous.restartStreak = plan.streak; + + if (plan.action === "limit") { + const first = plan.streak; + const summary = + `${SUPERVISOR_RESTART_LIMIT_TAG} service=${name} cause=${failureSignature(cause)} ` + + `attempts=${first.identicalRepeats} first-observed-at=${first.firstObservedAtIso ?? ""}` + + (first.firstCause ? ` first-cause=${first.firstCause}` : ""); + console.error(summary); + try { + await RunningProcess.appendLogBanner(previous.spawnOptions.stdoutLogFile, summary); + await RunningProcess.appendLogBanner(previous.spawnOptions.stderrLogFile, summary); + } catch (err) { + console.warn(`[DevNet][supervisor] failed to write restart-limit summary for '${name}': ${err}`); + } + return; + } - previous.restartCount += 1; - const attempt = previous.restartCount; - const delayMs = Math.min(30_000, 1_000 * Math.pow(2, Math.min(attempt - 1, 5))); - const cmdStr = previous.cmds.join(" "); - const ts = new Date().toISOString(); console.warn( `[DevNet][supervisor] process '${name}' EXITED (code=${code}, signal=${signal}, pid=${previous.pid}); ` + - `will RESTART in ${delayMs}ms (restart #${attempt}) cmd=${cmdStr}` + `will RESTART in ${plan.delayMs}ms (restart #${plan.restartCount})`, ); - await new Promise((r) => setTimeout(r, delayMs)); - if (this.stopping || previous.intentionalStop) { + const delay = Promise.withResolvers(); + setTimeout(delay.resolve, plan.delayMs); + await delay.promise; + if (!shouldScheduleSupervisedRestart(this.stopping, previous.intentionalStop)) { console.log(`[DevNet][supervisor] process '${name}' restart aborted (intentional stop or teardown)`); return; } + const first = plan.streak; + const firstPart = first.cause && first.firstObservedAtIso + ? ` first-failure=${failureSignature(first.cause)}@${first.firstObservedAtIso}` + : ""; + const firstCause = first.firstCause ? ` first-cause=${first.firstCause}` : ""; const banner = - `\n===== [DevNet supervisor] RESTART #${attempt} at ${ts} ` + + `\n===== [DevNet supervisor] RESTART #${plan.restartCount} at ${observedAtIso} ` + `(previous exit code=${code}, signal=${signal}, previous pid=${previous.pid}) =====\n` + `===== service: ${name} =====\n` + - `===== cmd: ${cmdStr} =====\n`; + `=====${firstPart}${firstCause} =====\n`; try { const restarted = await this.spawnFromTemplate(previous, banner, false); @@ -3805,32 +3896,38 @@ class DevNetProcessManager { console.log( `[DevNet][supervisor] process '${name}' RESTARTED successfully ` + - `(new pid=${restarted.pid}, restart #${attempt})` + `(new pid=${restarted.pid}, restart #${plan.restartCount})`, ); await RunningProcess.appendLogBanner( restarted.spawnOptions.stdoutLogFile, - `[DevNet][supervisor] process '${name}' is UP again pid=${restarted.pid} restart #${attempt}\n`, + `[DevNet][supervisor] process '${name}' is UP again pid=${restarted.pid} restart #${plan.restartCount}\n`, ); await RunningProcess.appendLogBanner( restarted.spawnOptions.stderrLogFile, - `[DevNet][supervisor] process '${name}' is UP again pid=${restarted.pid} restart #${attempt}\n`, + `[DevNet][supervisor] process '${name}' is UP again pid=${restarted.pid} restart #${plan.restartCount}\n`, ); if (name === "db") { this.restartProcessorsAfterDbRecovery(); } } catch (err) { console.error( - `[DevNet][supervisor] process '${name}' restart #${attempt} FAILED: ${err}` + `[DevNet][supervisor] process '${name}' restart #${plan.restartCount} FAILED: ${err}`, ); - if (!this.stopping && !previous.intentionalStop) { - const retryDelay = Math.min(60_000, delayMs * 2); + if (shouldScheduleSupervisedRestart(this.stopping, previous.intentionalStop)) { + const retryDelay = nextSpawnRetryDelayMs(plan.delayMs); console.warn( - `[DevNet][supervisor] will retry '${name}' again in ${retryDelay}ms (still counting as restart #${attempt})` + `[DevNet][supervisor] will retry '${name}' again in ${retryDelay}ms (still counting as restart #${plan.restartCount})`, ); - await new Promise((r) => setTimeout(r, retryDelay)); + const spawnRetry = Promise.withResolvers(); + setTimeout(spawnRetry.resolve, retryDelay); + await spawnRetry.promise; + if (!shouldScheduleSupervisedRestart(this.stopping, previous.intentionalStop)) { + console.log(`[DevNet][supervisor] process '${name}' spawn-retry aborted (intentional stop or teardown)`); + return; + } previous.hasExited = true; - previous.intentionalStop = false; - void this.handleSupervisedExit(previous, code, signal); + previous.healthySinceMs = 0; + void this.handleSupervisedExit(previous, code, signal, true); } } } @@ -3850,10 +3947,12 @@ class DevNetProcessManager { : await RunningProcess.spawn(template.cmds, options); process.name = template.name; process.restartCount = template.restartCount; + process.restartStreak = template.restartStreak; process.hintDetector = template.hintDetector; process.useInitHint = template.useInitHint; process.initMaxRetries = template.initMaxRetries; process.initRetryDelayMs = template.initRetryDelayMs; + if (!process.useInitHint) process.healthySinceMs = process.startedAtMs; return track ? this.track(process, template.name) : process; } private async waitForControlledStartDependency(process: RunningProcess): Promise { @@ -3918,6 +4017,10 @@ class DevNetProcessManager { const started: RunningProcess[] = []; try { for (const template of templates) { + template.restartCount = 0; + template.restartStreak = emptyRestartStreak(); + template.fatalProcessorErrorMarker = null; + template.firstCausalStderr = null; const process = await this.spawnFromTemplate(template); started.push(process); await this.waitForControlledStartDependency(process); diff --git a/docs/src/dev/bridge-common-operations.md b/docs/src/dev/bridge-common-operations.md index 59b303fe..67daf8cb 100644 --- a/docs/src/dev/bridge-common-operations.md +++ b/docs/src/dev/bridge-common-operations.md @@ -89,7 +89,7 @@ PURGE=1 make shutdown make run-all ``` -Keep `make run-all` in the foreground; it intentionally remains alive as the service supervisor (`Makefile:63-66`, `dev/locSetupV4.ts:5630-5635`). Run readiness checks and every later command from a second terminal whose current directory is ``. Do not background or abandon the supervisor. The supervisor truncates the relayer logs for this initial launch (`dev/locSetupV4.ts:649-658,4542-4549`), so the marker below belongs to the current process. Do not continue until all checks below succeed: +Keep `make run-all` in the foreground; it intentionally remains alive as the service supervisor (`Makefile:64-67`, `dev/locSetupV4.ts:5630-5635`). Run readiness checks and every later command from a second terminal whose current directory is ``. Do not background or abandon the supervisor. The supervisor truncates the relayer logs for this initial launch (`dev/locSetupV4.ts:649-658,4542-4549`), so the marker below belongs to the current process. Do not continue until all checks below succeed: ```bash curl -fsS http://127.0.0.1:3000/health @@ -484,7 +484,7 @@ rm -rf "$RESULT_DIR" make shutdown ``` -Do not manually kill individual services. A non-purge restart can separate L1 and L2 state; the walkthrough sections “33.10 locSetupV4 auto-restart timing” and “33.11 Non-purge devnet restart tears L1/L2 state apart” explain why routine cleanup uses `make shutdown` before the next `make run-all`. +Do not manually kill individual services. A non-purge restart can separate L1 and L2 state; the walkthrough sections “33.10 locSetupV4 auto-restart timing” and “33.11 Non-purge devnet restart tears L1/L2 state apart” explain why. Routine cleanup uses `make shutdown`, which purges the paired L1/L2 state by default (`PURGE ?= 1` in the Makefile), before the next `make run-all`; use `PURGE=0 make shutdown` only when the paired chain state must survive the cleanup. ## 16. Source and Walkthrough References diff --git a/docs/src/dev/devnet-launcher-reference.md b/docs/src/dev/devnet-launcher-reference.md index d987c2be..90248779 100644 --- a/docs/src/dev/devnet-launcher-reference.md +++ b/docs/src/dev/devnet-launcher-reference.md @@ -13,7 +13,7 @@ process construction, readiness, supervision, control commands, ports, persisten Use the Make targets for the repository-supported operating lifecycle, and use direct launcher commands only when selecting a documented component set or diagnosing launcher behavior. The lifecycle procedure remains owned by [devnet_lifecycle.md](devnet_lifecycle.md); this reference explains how the launcher implements that procedure -([devnet_lifecycle.md](devnet_lifecycle.md); `Makefile:60-79`; `dev/locSetupV4.ts:5435-5794`). +([devnet_lifecycle.md](devnet_lifecycle.md); `Makefile:61-80`; `dev/locSetupV4.ts:5435-5794`). ## Background @@ -22,7 +22,7 @@ readiness gates, process supervision, local Layer 1 persistence, and destructive (`dev/locSetupV4.ts:2747-2789`; `dev/locSetupV4.ts:3950-4839`; `dev/locSetupV4.ts:5218-5226`). Operators therefore need one source-grounded map that distinguishes supported Make commands from direct launcher mechanics, especially because a bare launcher invocation and `make run-all` select different counts, and because the embedded help contains -current-source defects that must not be copied into commands (`Makefile:60-66`; `dev/locSetupV4.ts:5443-5449`; +current-source defects that must not be copied into commands (`Makefile:61-67`; `dev/locSetupV4.ts:5443-5449`; `dev/locSetupV4.ts:5561-5566`; `dev/locSetupV4.ts:5581-5603`). ## Table of Contents @@ -234,17 +234,17 @@ make run-all `make run-all` does **not** rely on bare defaults. It expands an explicit component list with two Realms, two Coordinator workers, one Realm worker, one prove proxy, the database group, Coordinator/Realm nodes, faucet, Layer 1, -relayer stack, and all four UI flags (`Makefile:60-66`). It also defaults `PSY_SKIP_BRANCH_CHECK`, +relayer stack, and all four UI flags (`Makefile:61-67`). It also defaults `PSY_SKIP_BRANCH_CHECK`, `PSY_SKIP_KEYSTORE`, and `PSY_SKIP_BUILD` to `1`, whereas a direct launcher invocation has no intrinsic -`PSY_SKIP_BUILD=1` default (`Makefile:13-15`; `Makefile:66`; `dev/locSetupV4.ts:1933-1968`). +`PSY_SKIP_BUILD=1` default (`Makefile:13-15`; `Makefile:67`; `dev/locSetupV4.ts:1933-1968`). | Setting | Bare launcher | `make run-all` | Evidence | | ------------------- | -------------------------------------------------: | -------------------------------------: | ----------------------------------------------- | -| Realm count | 1 | 2 | `dev/locSetupV4.ts:5448-5486`; `Makefile:60` | -| Coordinator workers | 1 | 2 | `dev/locSetupV4.ts:5484`; `Makefile:60` | -| Realm workers | 2 | 1 | `dev/locSetupV4.ts:5477-5481`; `Makefile:60` | -| Prove proxy | Implicit 1 | Explicit 1 | `dev/locSetupV4.ts:4362-4369`; `Makefile:60` | -| Build policy | Build when required unless environment disables it | Existing artifacts required by default | `dev/locSetupV4.ts:1933-1968`; `Makefile:15,66` | +| Realm count | 1 | 2 | `dev/locSetupV4.ts:5448-5486`; `Makefile:61` | +| Coordinator workers | 1 | 2 | `dev/locSetupV4.ts:5484`; `Makefile:61` | +| Realm workers | 2 | 1 | `dev/locSetupV4.ts:5477-5481`; `Makefile:61` | +| Prove proxy | Implicit 1 | Explicit 1 | `dev/locSetupV4.ts:4362-4369`; `Makefile:61` | +| Build policy | Build when required unless environment disables it | Existing artifacts required by default | `dev/locSetupV4.ts:1933-1968`; `Makefile:15,67` | ### 4.3 Component Selection @@ -336,6 +336,38 @@ Core setup replaces child `PSY_CONFIG_PATH` with the generated public runtime co | `PSY_SKIP_KEYSTORE` | Direct default off; Make default `1` | Exact `1` skips remote trust-setup refresh/hash verification but still requires mandatory local files. | `dev/locSetupV4.ts:2487-2521`; `Makefile:14` | | `PSY_KEYSTORE_S3_BASE_URL` | Published development asset prefix | Overrides trust-setup manifest and proving-key download base. | `dev/locSetupV4.ts:1657-1660`; `dev/locSetupV4.ts:2285-2295` | +#### Using a Throwaway Bridge-Relayer Keystore + +`KEYSTORE_PATH` redirects only the bridge-relayer wallet; the proving trust setup stays under +`~/.psy/keystore` regardless of the value (`dev/locSetupV4.ts:2532`, `dev/locSetupV4.ts:2547`). This +exists so tests and CI can use a disposable relayer wallet without touching a developer's real +`~/.psy/keystore/bridge-relayer`. + +```bash +# Use a throwaway bridge-relayer keystore instead of ~/.psy/keystore/bridge-relayer +export WALLET_PASSWORD=devnet +export KEYSTORE_PATH=/tmp/psy-devnet-relayer/bridge-relayer-keystore +make run-all +``` + +Behavior on the first and later launches: + +1. Path absent — the launcher auto-generates a development keystore from the Anvil development + private key `0xac0974…2ff80`, encrypted with `WALLET_PASSWORD` (default `"devnet"` when unset) + (`dev/locSetupV4.ts:2390-2408`). +2. Path exists — the password must match that keystore. In a non-interactive session + (`WALLET_PASSWORD` unset, no TTY) startup fails with + `"WALLET_PASSWORD is required for an existing bridge-relayer keystore in a non-interactive + session"`; interactive sessions prompt instead (`dev/locSetupPolicy.ts:511-543`). An existing + keystore is never silently re-encrypted with the devnet default. +3. A mismatched password is not detected at startup validation alone — it surfaces later as a + decrypt failure (`invalid password`) when the relayer or deployment first uses the wallet + (`dev/locSetupPolicy.ts:546-555`). Always pass the same `WALLET_PASSWORD` for a given + `KEYSTORE_PATH`. + +The developer's real `~/.psy/keystore/bridge-relayer` is never read or modified while +`KEYSTORE_PATH` points elsewhere; trust-setup artifacts are not affected by this variable. + ### 6.4 Resources, Logging, and Supervision | Variable | Default | Effect | Evidence | @@ -351,7 +383,7 @@ Core setup replaces child `PSY_CONFIG_PATH` with the generated public runtime co | `SCYLLA_COMMITLOG_SYNC` | `batch` in foreground DB script | Sets foreground Scylla commitlog mode; daemon generation has no equivalent input. | `dev/start_db.sh:125-163`; `dev/locSetupV4.ts:4899-4908` | | `SCYLLA_COMMITLOG_BATCH_WINDOW` | `2` milliseconds | Sets foreground batch sync window. | `dev/start_db.sh:129-163` | | `SCYLLA_COMMITLOG_PERIOD` | `10` milliseconds | Sets foreground periodic sync interval. | `dev/start_db.sh:129-163` | -| `RUST_LOG` | No direct global default | Controls Rust tracing; Make maps `LOG_LEVEL` to `--env RUST_LOG=...`. | `Makefile:9,60`; `dev/locSetupV4.ts:3923-3930` | +| `RUST_LOG` | No direct global default | Controls Rust tracing; Make maps `LOG_LEVEL` to `--env RUST_LOG=...`. | `Makefile:9,61`; `dev/locSetupV4.ts:3923-3930` | | `PSY_NO_AUTO_RESTART` | Restart enabled | Exact `1` disables foreground child auto-restart. | `dev/locSetupV4.ts:3643-3645`; `dev/locSetupV4.ts:5775-5780` | | `TMPDIR` | `/tmp` | Bases the repository-keyed lock and control socket paths. | `dev/locSetupV4.ts:5238-5241`; `dev/locSetupV4.ts:5328-5330` | @@ -497,7 +529,7 @@ launch, uses the plan's `hasState` result, and passes the plan's reset decision (`dev/locSetupV4.ts:4404-4452`; `dev/locSetupV4.ts:4454-4475`). `make restart` sends a control command to the existing supervisor; it stops and recreates applications while the -tracked Anvil and DB processes remain alive (`Makefile:68-69`; `dev/locSetupV4.ts:3505-3512`; +tracked Anvil and DB processes remain alive (`Makefile:69-70`; `dev/locSetupV4.ts:3505-3512`; `dev/locSetupV4.ts:3851-3919`; `dev/locSetupV4.ts:5761-5772`). A non-purge shutdown stops Anvil but does not delete its state or the localhost deployment; a later launch loads the state and reuses the deployment (`dev/locSetupV4.ts:3450-3468`; `dev/locSetupV4.ts:2942-2992`). Purge deletes both `db/anvil` and the localhost @@ -513,9 +545,9 @@ queue, so lifecycle mutations do not overlap (`dev/locSetupV4.ts:5395-5413`). | Command | Supported Make target | Effect | Evidence | | ----------------- | ---------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| `restart` | `make restart` | Stop then start applications; keep DB and Anvil alive. | `Makefile:68-69`; `dev/locSetupV4.ts:3916-3919`; `dev/locSetupV4.ts:5763-5766` | -| `rollback-stop` | `make rollback-stop` | Stop applications, verify ports closed, write rollback sentinel. | `Makefile:71-72`; `dev/locSetupV4.ts:3851-3886` | -| `rollback-resume` | `make rollback-resume` | Start saved application templates and remove sentinel after success. | `Makefile:74-75`; `dev/locSetupV4.ts:3888-3913` | +| `restart` | `make restart` | Stop then start applications; keep DB and Anvil alive. | `Makefile:69-70`; `dev/locSetupV4.ts:3916-3919`; `dev/locSetupV4.ts:5763-5766` | +| `rollback-stop` | `make rollback-stop` | Stop applications, verify ports closed, write rollback sentinel. | `Makefile:72-73`; `dev/locSetupV4.ts:3851-3886` | +| `rollback-resume` | `make rollback-resume` | Start saved application templates and remove sentinel after success. | `Makefile:75-76`; `dev/locSetupV4.ts:3888-3913` | Application stop sends process groups `SIGTERM`, waits up to 15 seconds, escalates to `SIGKILL`, and verifies derived application ports are closed (`dev/locSetupV4.ts:3540-3587`; `dev/locSetupV4.ts:3604-3613`; @@ -542,9 +574,11 @@ for supervised restart so they rebuild infrastructure connections (`dev/locSetup ## 13. Teardown and Purge -`make shutdown` invokes `--teardown` and adds `--purge` only when `PURGE=1`; `make restart-all` performs purge shutdown -then `make run-all` (`Makefile:77-79`; `Makefile:102-106`). Direct teardown executes this sequence -(`dev/locSetupV4.ts:3402-3468`): +`make shutdown` invokes `--teardown --purge` by default: the Makefile sets `PURGE ?= 1` and the target adds `--purge` +unless an explicit `PURGE=0` limits it to bare `--teardown`; `PURGE=1 make shutdown` remains valid and is identical to +the default. Bare launcher `--teardown` stays non-purge regardless of the Make default. `make restart-all` performs +purge shutdown then `make run-all` (`Makefile:16`; `Makefile:78-80`; `Makefile:103-107`). Direct teardown executes +this sequence (`dev/locSetupV4.ts:3402-3468`): ```text 1. Kill known command patterns. @@ -590,13 +624,15 @@ make restart make rollback-stop make rollback-resume make shutdown +PURGE=0 make shutdown PURGE=1 make shutdown make restart-all ``` These are the repository-supported lifecycle entry points ([devnet_lifecycle.md](devnet_lifecycle.md); -`Makefile:63-79`; `Makefile:99-106`). `make restart` requires the original foreground supervisor; it is not a new -launcher startup (`Makefile:68-75`; `dev/locSetupV4.ts:5337-5369`). +`Makefile:64-80`; `Makefile:100-107`). `make shutdown` purges by default; `PURGE=0 make shutdown` is the +state-preserving form. `make restart` requires the original foreground supervisor; it is not a new +launcher startup (`Makefile:69-76`; `dev/locSetupV4.ts:5337-5369`). ### Bare foreground full mode @@ -709,11 +745,10 @@ These are limitations of the current source, not supported command examples. `dev/locSetupV4.ts:4923-5160`). 8. Daemonized Realm P2P uses a separate public-only runtime config: processor and edge public multiaddresses name the corresponding Compose DNS services, while each container listens on `/ip4/0.0.0.0/tcp/...`. -9. Teardown uses fixed process patterns and port ranges rather than the actual launch plan; Mode A port 5179 and a - hypothetical custom Anvil port are absent from the fixed listener list (`dev/locSetupV4.ts:3402-3447`). +9. Teardown uses fixed process patterns and port ranges rather than the actual launch plan; a hypothetical custom Anvil port is absent from the fixed listener list (`dev/locSetupV4.ts:3422-3470`). Mode A port 5179 is included in `killKnownPorts` and matched by `psy-dapp/mode-a-web-wallet-bridge` plus `vite.js --host 0.0.0.0 --port 5179`. 10. Startup purge with `--purge` performs the full paired teardown (processes, containers, checkpoints, Anvil state, deployments, volumes) before auto-setup; it is equivalent to `PURGE=1 make shutdown` followed by `make run-all` - (`dev/locSetupV4.ts:5693-5698`; `dev/locSetupV4.ts:3450-3468`). + (`dev/locSetupV4.ts:5693-5698`; `dev/locSetupV4.ts:3472-3492`). ## 18. Core Data Structures @@ -976,9 +1011,9 @@ Commands execute serially and the loop remains available until the server is clo [devnet_lifecycle.md](devnet_lifecycle.md)). 6. The control socket is local and permissioned `0600`, but any process running as the same account can attempt its three lifecycle commands (`dev/locSetupV4.ts:5317-5335`; `dev/locSetupV4.ts:5416-5424`). -7. Purge is intentionally destructive across both Layer 1 and Layer 2 state. Review the exact deletion set before - running `PURGE=1 make shutdown` or `make restart-all` (`Makefile:77-79`; `Makefile:102-106`; - `dev/locSetupV4.ts:3458-3466`). +7. Purge is intentionally destructive across both Layer 1 and Layer 2 state, and it is the `make shutdown` default. + Review the deletion set before running either command. Use `PURGE=0 make shutdown` to preserve state; + `make restart-all` always purges, even with `PURGE=0` (`Makefile:16`; `Makefile:78-80`; `Makefile:103-107`). 8. With Coordinator/Realm core selected, `--genesis-data-path` is input and output: startup rewrites validators. Use a disposable copy when preserving an existing validator list matters. Component-only modes without core processors do not rewrite it (`dev/locSetupV4.ts:1225-1234`; `dev/locSetupV4.ts:4072-4079`). diff --git a/docs/src/dev/devnet_lifecycle.md b/docs/src/dev/devnet_lifecycle.md index 23693fc4..0179c580 100644 --- a/docs/src/dev/devnet_lifecycle.md +++ b/docs/src/dev/devnet_lifecycle.md @@ -86,7 +86,7 @@ Do not weaken the provenance check. Do not add Cargo `[patch]` or `[replace]` ov ## 3. Fresh Start -A fresh test chain requires a full purge because Scylla volumes, checkpoint ring buffer files, the persisted Anvil snapshot, and localhost deployments must start from the same genesis state: +A fresh test chain requires a full purge because Scylla volumes, checkpoint ring buffer files, the persisted Anvil snapshot, and localhost deployments must start from the same genesis state. `make shutdown` purges by default (`PURGE ?= 1` in the Makefile); only `PURGE=0 make shutdown` preserves the paired state: ```bash PURGE=1 make shutdown @@ -97,7 +97,7 @@ PSY_SKIP_BUILD=1 \ make run-all ``` -`PURGE=1 make shutdown` removes checkpoints, `db/anvil/state.json`, logs, local deployments, and devnet Docker volumes. `make restart-all` performs this purge followed by `make run-all`. Never delete only Scylla, the Anvil state, deployments, or `local_checkpoints/`; retained components would describe different chains. +`PURGE=1 make shutdown` removes checkpoints, `db/anvil/state.json`, logs, local deployments, and devnet Docker volumes; bare `make shutdown` is identical because purge is the Make default. `make restart-all` performs this purge followed by `make run-all`. Never delete only Scylla, the Anvil state, deployments, or `local_checkpoints/`; retained components would describe different chains. `make run-all` stays in the foreground. Run it in a dedicated terminal or tmux pane. Do not background it with `&`. @@ -145,9 +145,9 @@ make restart `make restart` sends a command to the existing supervisor. It stops and recreates processors, edges, workers, proof services, APIs, indexers, relayer, and UIs from their recorded commands. The original Anvil process, database launcher, Scylla, Redis, NATS, Nostr, Envio containers, checkpoints, and logs remain active. Contract deployment and Envio storage initialization are not rerun. -Anvil continuously saves the local L1 chain to `db/anvil/state.json`. A non-purge `make shutdown` followed by `make run-all` loads that exact L1 state and reuses `psy-contracts/deployments/localhost/deployed-contracts.json`. The state and deployment must exist together; mismatch fails with an instruction to run `make restart-all`. +Anvil continuously saves the local L1 chain to `db/anvil/state.json`. A non-purge shutdown (`PURGE=0 make shutdown`) followed by `make run-all` loads that exact L1 state and reuses `psy-contracts/deployments/localhost/deployed-contracts.json`. The state and deployment must exist together; mismatch fails with an instruction to run `make restart-all`. -`make shutdown` is a full process teardown that preserves the paired Anvil state, L2 databases, checkpoints, and deployments. A later `make run-all` restores the same chain. For a complete fresh restart use: +`PURGE=0 make shutdown` is a full process teardown that preserves the paired Anvil state, L2 databases, checkpoints, and deployments. A later `make run-all` restores the same chain. Bare `make shutdown` purges by default and deletes that state; use it only when a fresh chain is intended. For a complete fresh restart use: ```bash make restart-all @@ -194,7 +194,7 @@ for p in 8545 9042 6379 4222 8081 5433 8080; do nc -z 127.0.0.1 "$p"; done make rollback-resume ``` -The supervisor removes the stop sentinel only after every saved application process reaches its startup condition. Never purge or run `make shutdown` between rollback plan generation, execution, recovery, and resume. +The supervisor removes the stop sentinel only after every saved application process reaches its startup condition. Never purge or tear down the stack — neither the default purging `make shutdown` nor the state-preserving `PURGE=0 make shutdown` — between rollback plan generation, execution, recovery, and resume. `make rollback-resume` restarts the saved application templates. It can take several minutes because release Plonky2 workers and the prove proxy rebuild circuit state. A long-running command is not stuck while new `CONTROLLED START` markers appear and readiness ports progressively open. The current supervisor starts templates serially; the prove proxy warm-up is usually the critical path. @@ -233,7 +233,7 @@ The earlier checkpoint-289-to-0 run proved L2 rollback, convergence, and transac ## 9. Forbidden Operations -- Do not use `make shutdown && make run-all` unless `db/anvil/state.json` and the localhost deployment both remain present. +- Do not use `PURGE=0 make shutdown && make run-all` unless `db/anvil/state.json` and the localhost deployment both remain present. Bare `make shutdown` purges by default and deletes both. - Do not run `docker compose` or manually restart one devnet service. - Do not run individual process binaries to substitute for the supervisor control targets. - Do not use `PSY_NO_AUTO_RESTART=1` when process-only restart or rollback stop/resume is required. diff --git a/docs/src/dev/processors.md b/docs/src/dev/processors.md index e608dbb9..78120c3a 100644 --- a/docs/src/dev/processors.md +++ b/docs/src/dev/processors.md @@ -143,6 +143,8 @@ A crash after the mapping and before the marker is `NeedsRecovery`. Recovery reb 5. `set_latest_checkpoint_id`. 6. `set_l2_latest_block_state` only after the marker. +Genesis C=0 is not inferred from the pending mapping. `new_init` writes C=0 DB records from the trusted bundle before constructing the backup manager when the complete singleton is missing and tip is 0. `ensure_genesis_applied` appends C=0 only when the backup ring is empty; a non-empty ring that still holds leaf 0 must match the trusted genesis hash. At tip 0, a missing mapping or incomplete L2 state replays C=0 records, and validator-tree genesis plus the `genesis_complete` singleton are written last. tip>0 does not rewrite C=0. Recovery rebuilds a cleared backup when local tip>0 and the ring is empty, and hard-resets an ahead ring when the coordinator tip is still 0. + `wait_for_realm_update_sync_with_coordinator` does not advance the singleton. The following `commit_state` does. ## 6. Sync and recovery diff --git a/docs/src/dev/realm-recovery-e2e.md b/docs/src/dev/realm-recovery-e2e.md new file mode 100644 index 00000000..24646a2c --- /dev/null +++ b/docs/src/dev/realm-recovery-e2e.md @@ -0,0 +1,606 @@ +# Realm Recovery E2E Runbook + +Internal developer runbook for the devnet E2E validation of the Realm recovery, +proposal-store, and P2P consensus pipeline. Not part of the published mdBook tree. + +Status: Review. Runtime observations below distinguish fresh evidence from +user-reported prior acceptance; source references describe checks, not test passes. + +## Environment + +Use the repository's configured devnet environment and launch `make run-all` +in the foreground under tmux, never with `&`. Do not copy credentials or +machine-specific environment paths into this runbook. Runtime operations are +owned by the session operator; the verification snippets below are read-only. + +Logs land in `logs/_.txt`. RPC surfaces: +coordinator edge `:1337`, realm edges `:13380/:13381` (realm 0) and +`:13390/:13391` (realm 1), faucet `:9998`, psy-services `:3000`, +prove proxy `:9999`, L1 anvil `:8545`. + +Snapshot logs before destructive reruns: + +```bash +archive_dir="$(mktemp -d)" +cp logs/realm_*processor_logs.txt logs/coordinator_edge_0_logs.txt "$archive_dir/" +``` + +## Logging prerequisite + +A prior run observed comma-separated `RUST_LOG` directives being truncated by +the launcher's `--env` parser (`LOG_LEVEL`, `Makefile:9`). Check the effective +child configuration when targets are missing; empty logs alone do not prove +that a follower accepted or rejected a proposal. The fresh run below contains +proposal, vote, certificate, and debug sync evidence; the prior logging +observation is not a fresh-run diagnosis. + +## Case 1 — normal block production plus transaction sequence + +Purpose: prove the full pipeline — EndCap intake, scheduled proposer, P2P +votes, certificate, coordinator inclusion, and per-checkpoint root agreement +across all four keyspaces. + +Steps: + +1. Launch the stack. Wait for readiness: + +```bash +grep -c "\[REALM_CREATE\] processor new done" logs/realm_0_sub_1_processor_logs.txt +``` + + Expect `>= 1` for each of the four `realm_{0,1}_{sub_1,sub_2}_processor` logs + (`create.rs:103`). + +2. Confirm the coordinator produces blocks: + +```bash +grep -c "Generated block in" logs/coordinator_processor_logs.txt +``` + + One line per checkpoint (`runner.rs:73`). Query all five tips to observe + progress; sequential tip reads can differ while the chain advances. Root + agreement must instead compare global roots at one fixed checkpoint: + +```bash +for p in 1337 13380 13381 13390 13391; do + curl -s -X POST http://127.0.0.1:$p -H 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"psy_get_latest_checkpoint_id","params":[]}' +done +``` + +3. Scenario F uses a **fresh registered user**, not genesis user 0. The + public faucet RPC still grants only 100 raw units and does not take an + amount. Fee-reserved funding used genesis user 0 calling contract 5 + `faucet(user_id, amount)` once, which deferred token `simple_transfer` + (`../psy-compiler/psy-precompiles/faucet/src/main.psy:17-45`). Amount was + `100 + 3 * 1_000_002_000` raw so the recipient's later `simple_claim` and + two successful `simple_transfer` EndCaps could pay + `GUTA_FEE + DA_FEE * slots` (`psy-genesis/config.json:65-66`; + `client_prover/psy_circuit/psy_ups_circuit/src/session.rs:1311-1338`). + Protocol fees were not changed. + +4. Recipient spendable balance requires `simple_claim(sender)` against the + faucet caller (`../psy-compiler/psy-precompiles/token/src/main.psy:445-465`). + Scenario F then used `psy_user_cli call ... --method-name simple_transfer` + for 50, 49, and rejected 10. `private-transfer` still exists, but that CLI + requires `--private-key` on argv (`args.rs:679-683`) and was not used. + +Judgment criteria (grep the proposer's and follower's processor logs): + +| Criterion | Greppable string | Source | +| --- | --- | --- | +| Proposal published | `realm P2P proposal published proposal=` | `process_block.rs:786` | +| Follower saved the body | body file appears under `local_checkpoints/realm_{R}_{S}/proposal_backups/bodies/` | `realm_p2p.rs:616` | +| Follower voted | `realm P2P vote published proposal= signer_sub_id=` (peer's sub id) | `drive.rs:221` | +| Proposer collected the vote | `realm P2P vote received proposal= signer_sub_id=` | `drive.rs:336` | +| Certificate formed | `realm P2P certificate formed proposal=` | `process_block.rs:853` | +| Submitted to coordinator | `Submitting GUTA proof to Coordinator proposal=` | `process_block.rs:606` | +| Coordinator admitted | `realm P2P certificate admitted realm=` | coordinator `handler.rs:889` | +| Inclusion applied | `Committed Realm proposal FFS checkpoint_id=` | `process_block.rs:397` | +| GUTA root changed | proposal `new_root != old_root` (body file name halves differ) | `replay_state_updates_into_tree` | +| 4th transfer rejected | `assertion failed: insufficient balance` from `simple_transfer`, not an RPC crash | `token/src/main.psy:423` | + +Failure signatures and what they mean: + +| String | Meaning | Where to look | +| --- | --- | --- | +| `dropped unauthenticated Realm vote` | a vote arrived for a proposal whose vote-auth context is not registered locally | `drive.rs:328`; expected on the follower for the proposer's self-vote, abnormal if the follower never votes | +| `follower baseline replay rejected proposal=` | the follower's baseline replay rejected the proposal (coverage or root binding) | `realm_p2p.rs:633`; reason comes from `recovery.rs` | +| `timed out: wait_votes` | proposer never reached the replication threshold; 120 s deadline | `runner.rs:122` | +| `InvalidStateUpdates: ... has no IMT record` | nonzero changed leaf on a tree classified as IMT-managed lacks its required IMT record; see limitations below, not a universal positional-write rejection | `psy_node_common/src/realm/processor/recovery.rs:389-393,570-603` | + +## H — lagging follower body catch-up + +Status: **passed on this stack after `aa6c252c`.** The earlier validator-leaf +restart loop is historical (`cf93c58c` coalescing + later preserved-state +docs). This run used the frozen C765 gap, not a new faucet grant. + +Frozen before resume: + +1. Realm-0 `sub_2` was SIGSTOP'd at tip 762, then SIGKILL'd; applications + stayed down through `make rollback-stop` with infrastructure retained. +2. Faucet grant `recipient_user_id=31` returned + `tx=85c6836ea13f53c8a6d74369b779aef87c1db72f1a4e5b49b4e9462c4dea6933`, + operator `786432`, `already_submitted=false`, request checkpoint 762. +3. Realm-0 `sub_1` certified proposal + `c3a316c118efab1aeace85baf4274860449b286245485777f8b9d87f6438e488` + (`signers=[1]`, base 763, target 764, pending 2399) and coordinator + included it at **C765**. Exact body name + `3927e571..._0e4e642f...` existed only under + `local_checkpoints/realm_0_1/proposal_backups/bodies/` (144231 bytes) + and was absent on `realm_0_2`. +4. Pre-fix follower recovery logged + `MissingHistoryProof at C=765: proof-base P=763 roots unavailable`. + +Fix and same-state resume: + +1. `aa6c252c` persists authenticated metadata for unchanged checkpoints + before verifying a later proof base (`init.rs:806-820`, + `require_checkpoint_metadata` in `sync.rs`). GPT and agentlo approved + the code; focused tests + `require_checkpoint_metadata_accepts_genesis_and_historical_proofs` and + `require_checkpoint_metadata_rejects_altered_bindings` passed. +2. Rebuilt release `psy_node_cli`, then `make rollback-resume` (not + `make run-all`). Follower local tip was still 762; coordinator tip 872. +3. `logs/realm_0_sub_2_processor_logs.txt` `14:58:06.988719Z` / + `14:58:06.992807Z`: + `Recovered unchanged Realm checkpoint metadata checkpoint_id=763` + and `...764`. +4. Same file `14:58:07.035320Z`: + `Committed coordinator processor state for checkpoint ID: 765`. + No post-resume `MissingHistoryProof at C=765`. Catch-up then continued + through the retained tip; `[REALM_STARTUP] reloaded gatherer tree after catch-up` + at `14:58:08.248172Z`. +5. The exact C765 body file appeared on `realm_0_2` at `14:58:06` with the + same 144231 bytes as `realm_0_1`. That is `proposal_store.install` of a + staged catch-up candidate (`init.rs:667-668`), not gossip + `Realm P2P proposal body complete` (that string is absent after resume). + Pre-resume `no verified candidate for pair=(3927e571...,0e4e642f...)` + is gone after the metadata persist. +6. `psy_get_checkpoint_global_state_roots [765]` is identical on + `:1337/:13380/:13381/:13390/:13391`. Realm-0 edges `:13380` and `:13381` + agree at C765: operator `786432` `psy_get_user_tree_leaf_hash` equals + `85c6836e...` (the faucet tx) and user `0` hashes match each other. + Realm-1 edges `:13390/:13391` have their own non-empty leaves for those + numeric ids; compare only same-realm edges, not across realms. + +Do not treat gossip `body complete` as the H peer-fetch proof. The +peer-fetch evidence here is: body absent on the follower before resume, +present and byte-identical after `install` of the catch-up staged file, +and C765 committed without ghost-pair retries. + +## Case 2 — proposer misses its epoch + +Purpose: pausing the scheduled proposer must not stall the coordinator +tip. Epoch is `target / CHECKPOINTS_PER_EPOCH` (`10` on localhost). +N=2 self-certification means `wait_votes` timeout is not required. + +Status: **closed on the H-resumed stack.** Two windows: + +1. Pause. Realm-0 `sub_1` PID 2472240 was `SIGSTOP`'d at coordinator + tip **962** (epoch 96) and `SIGCONT`'d at tip **995** (epoch 99). + While frozen, `:1337` advanced 962 → 995 without that processor. + After continue, five edges agreed at 998 then 1047; + `no peer offered pair` stayed 0 on both realm-0 processors. +2. Other sub_id. After recovery, a realm-0 faucet (`psy_claim_faucet` + on `:9998`, recipient 47, tx `5251c931…`, operator 393216, request + checkpoint 1147) produced the first realm-0 `sub_2` transaction + proposal of the run. + +Scheduled-proposer logs (`process_block.rs:744`) on this stack: + +| time | sub_id | epoch | target | note | +|---|---|---|---|---| +| 13:04:59Z | 1 | 76 | 764 | H faucet | +| 15:18:35Z | 1 | 96 | 963 | wake on pre-stop base, then catch-up to 997 | +| 15:23:11Z | 1 | 102 | 1025 | post-CONT transaction | +| 15:35:44Z | 1 | 110 | 1105 | later sub_1 epoch | +| 15:42:47Z | 2 | 114 | 1149 | first logged sub_2 takeover | + +The 15:42:47Z lines are: + +```text +realm P2P scheduled proposer realm=0 sub_id=2 epoch=114 target=1149 base=1148 +realm P2P certificate formed proposal=6dc6f946… realm=0 target=1149 epoch=114 signers=[2] verified_votes=1 +``` + +`sub_2` then committed realm checkpoint **1150** +(`Committed new realm block with checkpoint_id = 1150` at +15:42:54Z). Operator 393216 leaf hash on `:13380`/`:13381` stayed +`45371105…` through 1149 and became `5251c931…` at 1150 (equals the +faucet tx). Empty checkpoints do not print `scheduled proposer` +(`No GUTA jobs ... skipping` at `process_block.rs:446`); freeze-window +epochs that rotation assigned to `sub_2` were empty, so the takeover +log is after SIGCONT, not during the pause. Acceptance does not require +the other sub_id's transaction log while the pause is held. + +Do not treat wall-clock alone as rotation evidence. Epoch came from +checkpoint ids. JSON `random_seed` bytes are reversed relative to +in-memory Goldilocks limbs (`QHashOut` serde). + +## W — realm proving-worker freeze + +Purpose: freeze the observed realm proving worker without a TCP +disconnect. Coordinator CST workers stay running so checkpoints can +advance and rotation can change. Epoch is still +`target / CHECKPOINTS_PER_EPOCH`. + +Freeze-set: `psy_worker_cli` PID **2477876** +(`--completed-jobs-log-file ./local_checkpoints/realm_worker_0.backup`, +four `--realm-api-url` to `:13380/:13381/:13390/:13391`). +Leave running: coordinator workers 2476236 / 2477056 +(`coordinator_worker_{0,1}.backup`) and every `psy_node_cli` processor +and edge. + +| step | evidence | +|---|---| +| W0 | Five edges **1276**. Epoch 127 (anchor 1269) computed r0 `sub_1`, r1 `sub_2`. `logs/worker_0_logs.txt` size 35285; last live proving was RealmFinalizeGUTA goal 1148 submit ok. | +| W1 | SIGSTOP 2477876 at 16:04:11Z; `/proc` state `T`. Log size stayed 35285 for 8s (mtime 15:42:47Z). Coordinator workers remained `S`; tip 1279→1283. | +| W2 | Faucet recipient 48, tx `3673b143…`, operator 1441792, request C1284. Realm-1 `sub_1` logged `Realm worker publication acknowledged` at 16:05:09Z (`process_block.rs:118`) checkpoint 1286 unique_pending_id 4058, then no `waited for jobs` / `Persisted worker artifact ready`. Timeout is `u64::MAX` (`startup.rs:126`); `persisted_artifact.rs:16-17` skips the deadline. Classified unbounded silent await. Proofs not weakened. Second faucet recipient 49, tx `58f7ca64…`, operator 917504, request C1288. | +| W3 | Five `psy_get_checkpoint_global_state_roots[1286]` identical (`user_tree_root=cc23dad2…`). No post-16:04 Fatal / RESTART / channel-closed / preimage-mismatch. Live tips diverged only because r1s1 sat in the wait (`:13390` stayed 1286). | +| W4 | While 2477876 remained `T`, coordinator tip reached **1300** epoch **130**. Computed r0 `sub_2` / r1 `sub_2` from committed anchor 1299. W0 identity was r0 `sub_1` at epoch 127. Not wall-clock. | +| W5 | SIGCONT 2477876 at 16:07:41Z; state `T`→`S`; cmdline unchanged. `worker_0_logs.txt` grew past 35285. Post-CONT proving start/done/submit ok for GUTASingleEndCap goal **4058** and RealmFinalizeGUTA 1285/1290. r1s1 persisted the root proof at 16:07:45Z. | +| W6 | New proposer is realm-0 `sub_2`. Faucet recipient 51, tx `6f959773…`, operator 786432, request C1330. `16:12:17Z` `scheduled proposer realm=0 sub_id=2 epoch=133 target=1333`; certificate `signers=[2]`; follower r0s1 `proposal start accepted` + `proposal body complete` proposal=`9a079760…` body_len=143789; `Committed new realm block with checkpoint_id = 1334`. Operator leaf on `:13380`/`:13381` equals the tx at C1334; five-edge roots at 1334 identical. Freeze-window tx `58f7ca64…` operator 917504 first equals its leaf at C1306 on `:13380`/`:13381`; tx `3673b143…` operator 1441792 first equals its leaf at C1306 on `:13390`/`:13391`. Both are resume catch-up by epoch-129/128 `sub_1`, not the new proposer. | + + +## Case 3 — two proposers race the same target + +Purpose: verify an atomic single winner and a retryable loser without rollback. +Status: not executed; the literal two-author setup requires a revised injection +design before any test-tool changes. + +The coordinator permits exactly one scheduled proposer per realm and target; +a different author is rejected before the claim operation +(`psy_node_common/src/coordinator/edge/handler.rs:858-870`). Replaying a genuine +same-author request can exercise duplicate admission, but is not evidence of +two distinct legitimate proposers racing. + +The atomic claim is `put_submitted_status_if_absent`, keyed by coordinator +generation and submitting realm. A losing claim returns retryable +`AlreadyClaimed` (`psy_node_common/src/coordinator/edge/handler.rs:698-712`). +The submit error propagates before inclusion waiting and `commit_state` +(`psy_node_common/src/realm/processor/core/process_block.rs:611-651`); this +loser does not enter the foreign-root divergence path and needs no rollback. + +`realm P2P certificate admitted` is emitted during certificate verification, +before the atomic claim (`psy_node_common/src/coordinator/edge/handler.rs:681,698-712,888-895`). +Counting that log cannot prove a single winner. Acceptance needs the winning +queue publication, the loser's typed rejection and processor retry path, +no losing `commit_state` execution, and fixed-checkpoint root convergence. +A replay client alone cannot prove the processor retry branch. Report the +proposed injection and obtain approval before modifying any test tool. + +## Case 4 — restarts + +### 4a. Graceful restart (supervisor) + +`make restart` (control socket) or SIGTERM a child; the supervisor recreates +it with the saved template. Judgment: the restarted sub logs +`Recovering checkpoint N...` walking from `local_latest_checkpoint_id + 1` +(`init.rs:805`), reaches the tip, and its +`already synced to latest checkpoint ID` cadence resumes (`sync.rs:88`). +No `Local database is stale.` (`init.rs:491`) and no +`ahead of coordinator` (`init.rs:476`, `sync.rs:160`) lines. + +### 4b. Crash restart (kill -9) + +```bash +kill -9 +``` + +Judgment: + +- startup sweep removes unparseable residue from the ACTIVE bodies dir: + `remove_staged_files` deletes files whose name fails + `parse_transition_file_name` (`proposal_store.rs:374-385`) — plant + `.tmp--junk` and a garbage name before the kill; both must vanish. +- valid bodies survive: `_` files created before the crash are still + on disk after restart. +- damaged bodies (valid name, corrupt bytes) are deleted on hydrate — flip the + last byte of a body, restart, and the file disappears (damage path + `remove_transition`, `proposal_store.rs:242-256`). +- DB opens clean; no `Local database is stale.` / `ahead of coordinator`. + +### 4c. Full-network restart preserving state + +Executed: `PURGE=0 make shutdown` then `make run-all` with rebuilt +`psy_node_cli` at `5cc40680`. Relayer used a temporary `KEYSTORE_PATH`; HOME +relayer keystore was not overwritten. + +After restart, five surfaces returned identical +`psy_get_checkpoint_global_state_roots([452])` matching the pre-restart F +snapshot, including `user_tree_root=9f031366bb141a5ab2fde8cad4ada0b2322e99cc584d7b7ba402913e352f4e77`. +Realm-1 GUTA at last-modified checkpoint 452 remained +`578fa7dcb208e5e000a8fc737603d3b8d418eba9c8490aea429127969c4419fd`. +Tips had advanced (empty blocks); committed F roots did not regress. + +### 4d. Business continues after restart + +Executed after 4c, before attempting H: user 0 +`simple_transfer([1966080,1])` confirmed at checkpoint **600**, +`tx=f3207be4c2236ee409f8fda004a7aeab2333cbab566338c41ad18b39be5aa7a2`. +The transfer's own certificate was `signers=[1]` (proposal `857bd515...`). +User-0 leaf at 599 was still the C434 grant hash `3a681304...`; at 600 it +changed. Global roots at 600 matched on the five RPC surfaces then live. + +## Abandoned-directory rule + +The retired `local_checkpoints/realm_{R}_{S}/proposal_store/` directory must be +ignored: plant it with junk (`bodies/deadbeef_00c0ffee`), restart the sub, and +verify (a) zero log references to the old path and (b) the file survives. The +active path is `proposal_backups/bodies/` (`RealmProcessorStartConfig::get_proposal_backups_path`, +batch H). + +## Read-only verification of the fresh checkpoint + +These commands inspect the existing stack; they do not launch, restart, fund, +or transfer. They require `curl` and `jq`. Checkpoint 62 must still be retained. +Compare all six fields of the **global checkpoint roots** at checkpoint 62 on +all five surfaces. Different realms' local subtree roots are not expected to +equal each other. Latest-tip equality is not a substitute for this comparison. + +```bash +for p in 1337 13380 13381 13390 13391; do + curl --fail-with-body -sS "http://127.0.0.1:$p" \ + -H 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"psy_get_checkpoint_global_state_roots","params":[62]}' + printf '\n' +done | jq -s -e ' + if length == 5 and all(.[]; .error == null and (.result | type) == "object" and (.result | length) == 6) + then map(.result) | .[0] as $expected | if all(.[]; . == $expected) then . else error("global roots differ") end + else error("missing or invalid checkpoint roots response") end' +``` + +Inspect recipient user 0, contract 0, slot 0 at that same checkpoint on its +realm-0 edge; this is not the operator's realm-1 root: + +```bash +curl --fail-with-body -sS http://127.0.0.1:13380 \ + -H 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","id":2,"method":"psy_get_user_contract_state_tree_leaf_hash","params":[62,0,0,0]}' +``` + +Bind the included operator leaf to the claim response hash: + +```bash +for checkpoint in 61 62; do + curl --fail-with-body -sS http://127.0.0.1:13390 \ + -H 'content-type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"psy_get_user_tree_leaf_hash\",\"params\":[$checkpoint,1310720]}" + printf '\n' +done +``` + +## Session validation status (2026-09-16) + +**Prior acceptance, preserved as user-reported:** A, B, C, D, E, G (deletion +half), I, and 4b. These are not newly rerun passes. + +**Fresh run:** the operator rebuilt `psy_node_cli` and `psy_user_cli` in release +after `7877145b` (detached faucet task) and `6d28e2f6` (per-tree pairing), then +started a fresh devnet. A single long-lived recipient-0 claim returned +`amount="100"`, `operator_user_id=1310720`, request checkpoint 58, window 0, +`already_submitted=false`, and transaction hash +`c8f3393d5175f7682284ca186b8aadd8b7a23dbd6d4218ece13f270e8e0b1b0a`. +A repeat at checkpoint 64 returned the same hash with +`already_submitted=true`. These response observations are reported by the +session operator. No short-client-timeout claim probes ran on the fresh +stack. Long-lived claim success is **not cancellation end-to-end proof**. + +The checked log chain for proposal +`129fd57a55727ceeb7263e1321f5420f87184aa6ff5c39d6d6ac8ec37a0ab959` +is below. All timestamps are UTC on 2026-09-16. + +| Event | Timestamp / value | Current evidence | +| --- | --- | --- | +| Operator leaf lookup | 07:35:35.524137Z; user 1310720, checkpoint 59 | `logs/realm_1_sub_2_edge_0_logs.txt:2092` | +| Queue publish completed | 611 microseconds | `logs/realm_1_sub_2_edge_0_logs.txt:2103` | +| EndCap accepted | 07:35:35.529285Z; checkpoint 60; `872c109c3fd0b644e62928b159ca2b72afa103084fdf275967ccf22cd1379bd9` | `logs/realm_1_sub_2_edge_0_logs.txt:2105` | +| Gatherer dequeued operator EndCap | 07:35:35.540678Z; goal 102, `UserEndCap`, group 32, task index 1310720 | `logs/realm_1_sub_2_processor_logs.txt:7683` | +| EndCap job populated | 07:35:35.540804Z; user 1310720, checkpoint 58 | `logs/realm_1_sub_2_processor_logs.txt:7684` | +| Sub 2 published | 07:35:38.673377Z; base 59, target 60 | `logs/realm_1_sub_2_processor_logs.txt:7788` | +| Sub 1 accepted Start | 07:35:38.673848Z; 3 parts, 143561 bytes | `logs/realm_1_sub_1_processor_logs.txt:10413` | +| Sub 1 completed body | 07:35:38.673998Z | `logs/realm_1_sub_1_processor_logs.txt:10416` | +| Sub 2 received sub 1 vote | 07:35:41.806811Z | `logs/realm_1_sub_2_processor_logs.txt:7789` | +| Certificate formed | 07:35:41.813707Z; signers `[2, 1]` | `logs/realm_1_sub_2_processor_logs.txt:7791` | +| Inclusion confirmed | 07:35:54.871892Z; checkpoint 62 | `logs/realm_1_sub_2_processor_logs.txt:7816` | + +Group 32 alone does not identify a queue subject. The EndCap request checkpoint, +edge acceptance checkpoint, proposal target, and inclusion checkpoint are +distinct observations, not conflicting labels for one checkpoint. + +The session operator's RPC verification found the **operator realm-1 root**: + +- Before, checkpoint 58: `76985030cd9a34fc05636e59b590bf5b5fa0c38066dd58fff953cd447fff3c32`. +- After, checkpoint 62: `8c51952d7b24af61b93ad54a70e48d9d092fe8ed48cea3fdc4790c46e051e418`. +- All six fields of `psy_get_checkpoint_global_state_roots([62])` were + identical on ports 1337, 13380, 13381, 13390, and 13391. +- `psy_get_user_contract_state_tree_leaf_hash([62,0,0,0])` on port 13380 + remained raw `1_000_000_000_000_000`. This is recipient user 0's token + balance; the realm-1 root change is not evidence of a recipient root change. + +The operator's RPC reads also bind this specific EndCap to inclusion: +`psy_get_user_tree_leaf_hash([61,1310720])` on port 13390 was +`fc08f4575f01cd4d23e1aaf6d7f9586bda2b3eb8f3d8225bead35e8470d940bf`; +at `[62,1310720]` it was +`c8f3393d5175f7682284ca186b8aadd8b7a23dbd6d4218ece13f270e8e0b1b0a`, +exactly the claim response's transaction hash, not merely an unrelated +realm-root change. + +### F sequence (executed 2026-09-16) + +Fresh zk wallet registered as user **1966080** +(`public_key_hash=f5b5774c0c5dae283572149a9d8d93238b03169a9b4580ebf49ae17edb6a6552`). +Genesis user 0 called `faucet(1966080, 3000006100)`; inclusion +`tx=8d7efdff36a2ab0dfd6879a7bc416b7755e95ef1c0f9589bf1370a66a3102847` at +checkpoint **434**. Realm-0 user-0 leaf at 433 was +`5c9482a7c2fbf9c4188dd8a805c2aec148aec47098c8e52ddad4435cf2b4193c`; at 434 it +matched the grant EndCap hash +`3a68130440bda5e26bb9cdec70eff355679b8ef52b969cc490a74f5c95fdca21`. + +| Step | Checkpoint | tx / result | User 1966080 contract-0 slot 0 raw | +| --- | --- | --- | --- | +| After grant, before claim | 434 | operator grant only | `0` | +| `simple_claim([0])` | 442 | `06f295093c715661dd22da53c04534b33b0f1492d0f084a99f6885ea8fbe79d4` | `2000004100` | +| `simple_transfer([1,50])` | 448 | `76f1f058aeab5878ae9176f25d55d11a20508c6b2f811fa3c368efe0d5c8835c` | `1000002050` | +| `simple_transfer([1,49])` | 452 | `05b36954a5d7c6e0410747b1297073a5224e94c62342e257386892c475557203` | `1` | +| `simple_transfer([1,10])` | not submitted | CLI error `assertion failed: insufficient balance (left: 0, right: 1)` | still `1` | + +Recipient EndCap hashes at inclusion: + +- C442 `7c55ca8e15b5a8c4f0a0d055494680216794e8306b223ef11045bbede43153d2` +- C448 `d8bde3c9f2ef7dfa64a446a1192a5dd16a91b5ee5ec89a3934394afb48f8c46c` +- C452 `f7371c9dc162d22837cda9c764d60080eda59d9f43bd186c94500f768d143b48` + +`psy_get_checkpoint_global_state_roots` at **434, 442, 448, and 452** returned +identical six-field objects on ports 1337, 13380, 13381, 13390, and 13391. + +Observed burn per recipient EndCap was `1_000_002_000` raw (claim left +`2000004100`; first transfer left `1000002050`; second left `1`). + +**F is passed under the fee-reserved fresh-account interpretation.** It is not +a pass of raw faucet RPC amount 100 covering three EndCaps, and it is not a +`private-transfer` pass. + +Pending: **H → Case 2 → Case 3**. 4c/4d passed. H and Case 2 remain blocked +because the preserved validator user-leaf row does not hash to the +persisted tree leaf. `cf93c58c` is not validated against a fresh +`signers=[1]` inclusion. Case 3 still cannot create two distinct scheduled +proposers without a harness change. + +### Accepted limitations, not fixes + +- The approximately 105 MB / 1711 gossip-parts case remains an accepted known + limitation, not fixed. The fresh proposal's 143561-byte / 3-part body does + not exercise it. +- Two-validator realms accept a certificate signed by the proposer alone + (`ceil(2 / 2) = 1`). This gives up "no unilateral certification" at + `n == 2` in exchange for liveness under a single-node fault. Reachable + only when a realm has exactly two validators; at `n >= 3` the + `ceil(n / 2)` threshold already forces a non-proposer signer, so this is + a no-op for production realms. +- Per-tree pairing classifies a tree using a **positive LIVE next-append + pointer**, then requires IMT records for its nonzero changed leaves, + including new indices on an already-managed tree + (`psy_node_common/src/realm/processor/recovery.rs:570-603,389-393`). + Zero-valued changed leaves remain exempt. The pointer read is not + checkpoint-versioned + (`psy_node_core/src/psy_core_db/v3_implementation/full.rs:4142-4151`). + A completely zero-pointer tree's first insertion remains exempt from this + pairing classification. This is not a claim of universal IMT soundness. + +## Post-squash rerun (2026-09-16) + +Stack: existing H-resumed chain, no PURGE. HEAD `47ce6a33` after +thematic squash (`backup/pre-squash-8b7b2e8e` tree +`5ecf0d24` identical). RPC dumps live under local gitignored +`e2e-evidence/round1-*`. Case 3 not run. + +### F — fee-reserved sequence — PASS + +Fresh zk wallet registered as user **1966080** +(`public_key_hash=b3ad81eb…`, `e2e-evidence/round1-f/get-user-id.json`). Genesis user 0 called +`faucet(1966080, 3000006100)`; confirmed C**1597** +`tx=d9162c00…` (`grant.json`). Recipient `simple_claim([0])` C**1603** +`tx=b2ceadb7…`. `simple_transfer([1,50])` C**1608** `tx=2853b540…`. +`simple_transfer([1,49])` C**1613** `tx=b2ab6b51…`. Fourth +`simple_transfer([1,10])` failed at trace: +`assertion failed: insufficient balance (left: 0, right: 1)` +(`xfer10-reject.stderr.txt`). Five-edge +`psy_get_checkpoint_global_state_roots` equal at 1597, 1603, 1608, +1613 (`*-rpc.json`). Pipeline grep: +`e2e-evidence/round1-f/pipeline-grep.txt`. + +### H — lagging follower body catch-up — PASS + +SIGSTOP r0s2 PID **2474005** at 16:59:08Z tip **1623** (`freeze.json`, +`/proc` `T`). User-0 `simple_transfer([1,1])` CLI timed out waiting +inclusion, but r0s1 published proposal `570580d5…` epoch 163 target +1633 at 17:00:44Z and committed C**1635**. Operator/user-0 leaf on +`:13380` stayed `95e5b506…` through 1634 and became `e910b259…` at +1635 (`inclusion-rpc.json`). While frozen, `:13381` stayed 1623. +SIGCONT 17:04:22Z (`cont.json`). r0s2 logged +`Realm P2P proposal body complete` proposal=`570580d5…` then +`Committed coordinator processor state for checkpoint ID: 1635`. +Body file +`a6170ddb…_74feba83…` (140880 bytes) appeared on both r0s1 (mtime +17:00:44Z) and r0s2 (mtime 17:04:22Z); `cmp` identical +(`body-cmp.json`). Five-edge roots at 1635 identical; `no peer +offered pair` stayed 0. Gossip `body complete` is the follower +receipt; install is the 17:04:22Z body file plus C1635 commit. + +### Case 2 — missed epoch nonempty takeover — FAIL + +SIGSTOP r0s1 PID **2472240** at 17:06:14Z tip **1665** epoch **166** +(computed r0 `sub_1`). Coordinator tip advanced 1665→**1679+** while +that processor stayed `T`; `:13380` stayed 1665; ghost 0 +(`freeze.json`, `epoch-scan.json`). Epoch **168** (anchor 1679) computed +r0 `sub_2` (`epoch-scan.json` last sample 01:08:03, tip 1679, +`r0_168=2`). r0s2 logged only +`No GUTA jobs to process in this block, skipping.` (`process_block.rs:446`); +last `scheduled proposer realm=0 sub_id=2` remains 16:54:33Z epoch 159 +(`r0s2-scheduled.txt`). Faucet returns on `:9998` used stale request +checkpoint **1665** (frozen r0s1 edge). User-0 EndCap hit +`already been submitted` unique_pending_id 5174 on `:13380`. SIGCONT +17:16:53Z; five edges later 1742. **Tip continued is not rotation +takeover.** Nonempty sub_2 proposal during the pause was not obtained. + + +### W — realm proving-worker freeze — PASS + +| step | evidence | +|---|---| +| W0 | Five edges **1746**. Epoch 174 (anchor 1739) computed r0 `sub_2`, r1 `sub_2`. Realm worker PID **2477876** `S`; `logs/worker_0_logs.txt` size 68357. Coordinator workers 2476236/2477056 `S`. `e2e-evidence/round1-w/w0.json`. | +| W1 | SIGSTOP 2477876 at 17:17:42Z; `/proc` `T`. Size stayed 68357 for 8s. Coordinator workers `S`; tip 1746→1747 (`w1.json`). | +| W2 | Faucet recipient 55, tx `b68c038e…`, operator 655360, request C1749 (`w2-faucet.json`). r0s1 `Realm worker publication acknowledged` 17:18:30Z checkpoint 1751 unique_pending_id **5206** (`w2-ack.txt`); no `waited for jobs` before SIGCONT. Timeout `u64::MAX` (`startup.rs:126`). Unbounded silent await. Proofs not weakened. | +| W3 | Five `psy_get_checkpoint_global_state_roots[1746]` identical (`user_tree_root=daa3bc6a…`). Live tips during wait: `:13380`=1751 others 1753 (r0s1 blocked). No 17:17–17:19 Fatal/RESTART/channel-closed/preimage-mismatch (`w3-rpc.json`, `w3-fatals.txt`). | +| W4 | W0 identity r0 `sub_2` epoch 174. While worker `T`, tip 1753 epoch **175** computed r0 `sub_1` from committed anchor 1749 (`w4-scan.json`). Not wall-clock. | +| W5 | SIGCONT 17:19:14Z; `T`→`S`; `size_before=68357` (`w5-cont.json`). `logs/worker_0_logs.txt` then contains `proving start` goal **5206** GUTASingleEndCap (`w5-proving.txt`). | +| W6 | New proposer r0 `sub_1` epoch 175. 17:19:16Z `scheduled proposer realm=0 sub_id=1 epoch=175 target=1751`; certificate `signers=[1]`; follower r0s2 `proposal start accepted` + `proposal body complete` proposal=`0c2868a7…` body_len=**143789**; `Committed new realm block with checkpoint_id = 1757`. Operator 655360 leaf on `:13380` first equals tx `b68c038e…` at C1757 (C1756 still `8ce29ee7…`). Five-edge roots at 1757 identical (`w6-rpc.json`). | + +### 4c — PURGE=0 full-network restart — PASS + +Frozen checkpoint **1915** before teardown (`e2e-evidence/round2-4c/pre-summary.json`). +Five tips **1938**. `user_tree_root=367179a0bcebb17b…`, +`gutas_root=712d7ea35db8ce89…`. User-0 leaf `:13380/:13381` +`b667ba9a…`; `:13390/:13391` `5c9482a7…` (same-realm only). +L1 `eth_blockNumber=0x1a37`, StateManager `0x93df5526…` +(`pre-l1.json`). `PURGE=0 make shutdown` then +`PSY_SKIP_BUILD=1 PSY_SKIP_BRANCH_CHECK=1 PSY_SKIP_KEYSTORE=1 make run-all` +in the foreground. After restart: +`[COORD_CREATE] processor new done` 17:50:55.455843Z; +`[REALM_CREATE] processor new done` r0s1 17:52:34, r1s1 17:53:09, +r0s2 17:53:44, r1s2 17:54:19 (`*-create.txt`). Five tips **1974** +then continuing. C1915 roots, GUTA, user-0 leaves, L1 addresses, and +`eth_blockNumber` unchanged (`post-summary.json`, `post-rpc.json`). +Anvil reused `db/anvil/state.json`; launcher printed +`Reusing persisted localhost deployment`. No +`Local database is stale.` / `ahead of coordinator`. + +### 4d — post-restart transfer — PASS + +User 0 `call` contract 0 `simple_transfer([1966080,1])` after +`[CFLI:PSY_PROVE_PROXY_STARTED][0.0.0.0:9999]`. First EndCap +`tx=dc0e026d…` `end_user_leaf_hash=9117c287…` submitted 18:00:53Z +and CLI timed out at latest 2018 (`xfer-key.txt`, `xfer.stderr.txt`). +That hash never appeared on `:13380`. User-0 leaf instead changed at +C**2001** to `82f1237e…` (nonce 5→6, balance 22000→24000, +`last_checkpoint_id` 1750→1999; `competing-leaf.json`). Not replayed. + +Second call from the C2001 leaf confirmed C**2036** +`tx=baee8fa164b7b193…` `end_user_leaf_hash=8b7afa0d60a226af…` +(`xfer2.json`). r0 `sub_2` epoch 203 target 2035 published proposal +`3b76bb9f…`, certificate `signers=[2]`; r0s1 +`proposal start accepted` + `proposal body complete` + +`vote published` then `Applied Realm gatherer FastForward` at C2036; +r0s2 `Committed new realm block with checkpoint_id = 2036` +(`w-commit.txt`). Operator/user-0 leaf on `:13380/:13381` stayed +`82f1237e…` at 2035 and became `8b7afa0d…` at 2036 (nonce 6→7) +(`inclusion-rpc.json`). Five-edge +`psy_get_checkpoint_global_state_roots([2036])` identical. +C1915 `user_tree_root` still `367179a0…` after 4d +(`inclusion-rpc.json`). Recipient 1966080 lives on +realm 1; its user leaf on `:13390/:13391` stayed `0e49b588…` at both +C2035 and C2036 (`inclusion-rpc.json`; token contract leaf already +`…0001` from the earlier F transfers). + diff --git a/docs/src/dev/rollback-operations.md b/docs/src/dev/rollback-operations.md new file mode 100644 index 00000000..d67d7d9d --- /dev/null +++ b/docs/src/dev/rollback-operations.md @@ -0,0 +1,298 @@ +# Local Rollback Operations + +Status: Review. Updated: 2026-09-16. Internal operator runbook. + +## Terminology + +- **RP**: one role-local rollback plan, serialized as JSON (JavaScript Object Notation). +- **CLI**: command-line interface. +- **L1 / L2**: Layer 1 / Layer 2. +- **P2P**: peer-to-peer Realm transport. +- **YAML**: YAML Ain't Markup Language, the processor template format. +- **E2E**: end-to-end verification with a real committed state change. +- **JTMB**: the test-only “just trust me bro” proving backend; forbidden here. + +## Scope and authority + +Use five independent RPs to roll back the default local stack: one Coordinator and four Realm processors. This runbook supplies executable generation, inspection, and execution commands; [devnet lifecycle, sections 6–9](devnet_lifecycle.md#6-offline-rollback-stop-and-resume) owns stopping, retained infrastructure, resuming, and acceptance. [Launcher control lifecycle](devnet-launcher-reference.md#11-control-socket-and-application-lifecycle) explains the supervisor. Do not substitute a new launcher invocation for the saved supervisor commands (`docs/src/dev/devnet_lifecycle.md:164-243`; `Makefile:61-76`). + +This procedure requires the default foreground stack on loopback, realms 0 and 1, two validators per Realm, one edge per validator, and one Coordinator edge. It does not cover a custom topology, daemonized stack, or remote network (`Makefile:61`; `dev/locSetupV4.ts:1004-1007,1125-1135,4195-4206,4224-4239,4293-4317`). + +**Never follow the current rollback CLI success text that suggests `make run-all`. Use only `make rollback-resume` after every RP and external recovery are complete.** The text is stale relative to the canonical lifecycle (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:139-156`; `docs/src/dev/devnet_lifecycle.md:190-199`). + +## Table of contents + +- [1. Flow](#1-flow) +- [2. Preflight and shell variables](#2-preflight-and-shell-variables) +- [3. Processor configs and role matrix](#3-processor-configs-and-role-matrix) +- [4. Stop and generate all five RPs](#4-stop-and-generate-all-five-rps) +- [5. Inspect and validate the frozen RPs](#5-inspect-and-validate-the-frozen-rps) +- [6. Execute and require completion](#6-execute-and-require-completion) +- [7. Failure and resume procedure](#7-failure-and-resume-procedure) +- [8. Resume and post-E2E acceptance](#8-resume-and-post-e2e-acceptance) +- [9. Security](#9-security) + +## 1. Flow + +```mermaid +sequenceDiagram + participant Operator + participant Supervisor + participant CLI as Release psy_dev_cli + participant Stores as Retained stores + Operator->>Supervisor: 1. make rollback-stop + Supervisor-->>Operator: 2. Applications stopped; sentinel written + loop Coordinator and four Realm identities + Operator->>CLI: 3. --generate with unique RP path + CLI->>Stores: 4. Read target, mappings, backups, high-water + CLI-->>Operator: 5. Validated RP persisted + end + Operator->>Operator: 6. Inspect all five frozen RPs + loop Coordinator and four Realm identities + Operator->>CLI: 7. --execute with the same RP + CLI->>Stores: 8. Apply phases, verify, commit marker last + CLI-->>Operator: 9. Persist completed phases + end + Operator->>Operator: 10. Require all phases completed and external recovery complete + Operator->>Supervisor: 11. make rollback-resume + Supervisor-->>Operator: 12. Saved applications ready; sentinel removed +``` + +Generation reads authoritative stores and backups; execution writes the checkpoint marker last and persists phase progress (`psy_cli/psy_dev_cli/src/subcommand/rollback/generate.rs:301-340`; `psy_node_common/src/rollback/executor.rs:130-203`). + +## 2. Preflight and shell variables + +1. Complete the existing [startup preflight](devnet_lifecycle.md) before this maintenance window: release binaries and matching Plonky2 artifacts must already exist. Do not rebuild, regenerate Genesis, rotate validator keys, or change configuration during rollback (`AGENTS.md:25-41`). +2. Record the L1 block number, StateManager/Bridge/Router addresses, L1 finalized checkpoint, all five L2 heads, and the target application state before stopping. Choose one target checkpoint present in every processor's retained history; generation rejects a target newer than that processor's head (`docs/src/dev/devnet_lifecycle.md:168,201-210`; `psy_cli/psy_dev_cli/src/subcommand/rollback/generate.rs:355-368`). +3. Keep the original foreground supervisor alive. The following blocks run in **one Bash operator shell from the repository root**, not in the supervisor's terminal. Bash, `jq`, `sed`, `cp`, `mkdir`, `make`, and the already-built release CLI are prerequisites. Shell checks here are operator gates, not a replacement for Rust plan validation. + +Set variables once. `REPO_ROOT` identifies the current checkout; `WORKSPACE` is a new operator-selected relative directory outside `local_checkpoints`; `TARGET` is the recorded target checkpoint; `RP_DIR` holds the five RPs. For example, enter `rollback-work-001` for the workspace. The workspace must not already exist, preventing accidental replacement of a frozen RP set. + +```bash +set -euo pipefail +umask 077 +REPO_ROOT="$PWD" +test -f "$REPO_ROOT/Makefile" +test -f "$REPO_ROOT/psy_cli/psy_dev_cli/src/subcommand/rollback.rs" +read -r -p 'New relative rollback workspace directory: ' WORKSPACE +[[ "$WORKSPACE" =~ ^[A-Za-z0-9_-]+$ ]] +[[ "$WORKSPACE" != local_checkpoints ]] +read -r -p 'Recorded target checkpoint: ' TARGET +[[ "$TARGET" =~ ^(0|[1-9][0-9]*)$ ]] +CLI="$REPO_ROOT/target/release/psy_dev_cli" +RP_DIR="$WORKSPACE/plans" +CONFIG_DIR="$WORKSPACE/config" +STOP_SENTINEL=local_checkpoints/rollback-stop.sentinel +export PSY_CONFIG_PATH=./local_checkpoints/realm_p2p/config.json +export PSY_NETWORK=localhost +test -x "$CLI" +test -r "$PSY_CONFIG_PATH" +test -r genesis.json +command -v jq +mkdir "$WORKSPACE" +mkdir "$RP_DIR" "$CONFIG_DIR" +printf '%s\n' "$TARGET" > "$WORKSPACE/target-checkpoint.txt" +``` + +`PSY_CONFIG_PATH` must remain the launcher's generated **public runtime config**, not `psy-genesis/config.json` and not a processor config. It supplies the validator identities used to derive each Realm sub-identity. The launcher exports this exact path and selects `localhost`; rollback derives the sub-identity from the existing local identity key (`dev/locSetupV4.ts:1004,1180-1182,4180-4184`; `psy_cli/psy_dev_cli/src/subcommand/rollback.rs:237-257`; `psy_cli/psy_node_cli/src/node/realm_p2p.rs:359-382`). + +## 3. Processor configs and role matrix + +**There is no launcher-generated `locSetupV4/config` directory.** The current launcher starts processors with arguments, not processor config files. The block below copies the checked-in Coordinator template and derives four operator-owned Realm YAML configs from the two checked-in sub-1 templates. It changes only namespace, key paths, and listener to match the default launcher. It never edits the templates or launcher output (`dev/locSetupV4.ts:4195-4206,4305-4317`; `psy_cli/example_node_configs/coordinator_processor_1.yaml:1-12`; `psy_cli/example_node_configs/realm_0_processor.yaml:1-18`; `psy_cli/example_node_configs/realm_1_processor.yaml:1-18`). + +| RP filename under `RP_DIR` | `--role` | Processor config under `CONFIG_DIR` | Identity in config | Required Realm flags | `db_namespace` | RP identity fields | +|---|---|---|---|---|---|---| +| `coordinator.json` | `coordinator` | `coordinator.yaml` | `coordinator_id: 0`, `coordinator_sub_id: 0` | Neither flag | `coordinator` | `realm_id: 0`, `realm_sub_id: 0` | +| `realm0sub1.json` | `realm` | `realm0sub1.yaml` | `realm_id: 0`, sub 1 identity key | `--realm-id 0 --realm-sub-id 1` | `realm_0_1` | `0`, `1` | +| `realm0sub2.json` | `realm` | `realm0sub2.yaml` | `realm_id: 0`, sub 2 identity key | `--realm-id 0 --realm-sub-id 2` | `realm_0_2` | `0`, `2` | +| `realm1sub1.json` | `realm` | `realm1sub1.yaml` | `realm_id: 1`, sub 1 identity key | `--realm-id 1 --realm-sub-id 1` | `realm_1_1` | `1`, `1` | +| `realm1sub2.json` | `realm` | `realm1sub2.yaml` | `realm_id: 1`, sub 2 identity key | `--realm-id 1 --realm-sub-id 2` | `realm_1_2` | `1`, `2` | + +A Realm processor config has **no `realm_sub_id` field**: unknown fields are rejected, and rollback derives that value from the key and runtime validator registry. The command-line value must match. Coordinator identity is stored in the RP's `realm_id`/`realm_sub_id` fields but must not be supplied through Realm CLI flags (`psy_node_core/src/config/node_cli_config.rs:34-114,191-254`; `psy_cli/psy_dev_cli/src/subcommand/rollback.rs:173-195,237-270`; `psy_node_common/src/rollback/validate.rs:31-33`). + +```bash +cp psy_cli/example_node_configs/coordinator_processor_1.yaml "$CONFIG_DIR/coordinator.yaml" +for realm in 0 1; do + for sub in 1 2; do + port=$((41000 + realm * 20 + sub)) + for suffix in processor_identity.key bls.key zk.key; do + test -r "./local_checkpoints/realm_p2p/realm_${realm}_sub_${sub}_${suffix}" + done + sed \ + -e "s/^db_namespace: realm_${realm}_1$/db_namespace: realm_${realm}_${sub}/" \ + -e "s/realm_${realm}_sub_1_/realm_${realm}_sub_${sub}_/g" \ + -e "s|^p2p_listen: .*|p2p_listen: /ip4/127.0.0.1/tcp/${port}|" \ + "psy_cli/example_node_configs/realm_${realm}_processor.yaml" \ + > "$CONFIG_DIR/realm${realm}sub${sub}.yaml" + done +done +``` + +The substitutions match the checked-in template lines above; namespaces, ports, and key paths are defined at `dev/locSetupV4.ts:1120-1127,1342-1347`. Connection values match `dev/locSetupV4.ts:3675-3690`. The loader chooses YAML by extension and requires the typed processor fields (`psy_node_core/src/config/node_cli_config.rs:8-16,34-114,191-254`). Section 4's exact `--generate` commands are the config validation gate: before reading rollback stores or writing an RP they load the config, derive identity, enforce local-devnet and matching Realm flags, then verify offline guards (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:125-138,163-170,198-257`). There is no separate config-only validation option in the rollback clap definition (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:21-74`). + +The backup root remains `./local_checkpoints`, resolving to `coordinator_0_0` and `realm_0_1`, `realm_0_2`, `realm_1_1`, `realm_1_2`. Never point it at the RP workspace. These directories contain checkpoint-tree and gatherer backups, not RPs (`psy_node_core/src/config/node_start_config.rs:37-53,98-127`). + +## 4. Stop and generate all five RPs + +Follow lifecycle section 6 to stop applications and check retained infrastructure. In this operator shell: + +```bash +make rollback-stop +test -f "$STOP_SENTINEL" +``` + +Do not create the sentinel manually. The supervisor owns its exact content and must confirm application ports are closed (`docs/src/dev/devnet_lifecycle.md:169-188,226`; `docs/src/dev/devnet-launcher-reference.md:546-558`). + +Define the common arguments and transparent role functions. Every invocation probes **all five default edge endpoints**, including Coordinator generation. The Coordinator endpoint is always required; Realm endpoints and both identity flags are required for Realm plans. These are reachability guards, not a substitute for stopping processors and the relayer (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:50-73,173-195,273-322`; default ports: `dev/locSetupV4.ts:1132-1135,4224-4239,4341-4345`). + +```bash +COMMON=( + --target "$TARGET" + --proving-backend plonky2-poseidon-goldilocks + --stop-sentinel "$STOP_SENTINEL" + --coordinator-endpoint http://127.0.0.1:1337 + --realm-endpoint http://127.0.0.1:13380 + --realm-endpoint http://127.0.0.1:13381 + --realm-endpoint http://127.0.0.1:13390 + --realm-endpoint http://127.0.0.1:13391 +) +rollback_coordinator() { + "$CLI" rollback "$@" "${COMMON[@]}" \ + --role coordinator \ + --processor-config "$CONFIG_DIR/coordinator.yaml" \ + --rp-path "$RP_DIR/coordinator.json" +} +rollback_realm() { + local realm="$1" sub="$2" + shift 2 + "$CLI" rollback "$@" "${COMMON[@]}" \ + --role realm --realm-id "$realm" --realm-sub-id "$sub" \ + --processor-config "$CONFIG_DIR/realm${realm}sub${sub}.yaml" \ + --rp-path "$RP_DIR/realm${realm}sub${sub}.json" +} + +rollback_coordinator --generate --reward-realm-id 0 --reward-realm-id 1 +rollback_realm 0 1 --generate +rollback_realm 0 2 --generate +rollback_realm 1 1 --generate +rollback_realm 1 2 --generate +``` + +**Finish all five generation commands before any execution.** Coordinator generation requires the explicit nonempty reward Realm set; Realm generation uses its own Realm automatically. `--reward-realm-id` belongs only to generation (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:23-32`; `psy_cli/psy_dev_cli/src/subcommand/rollback/generate.rs:485-518`). + +These commands intentionally omit `--target-contract-state`. It is optional, generation-only, and retains a snapshot only when `last_finalized_checkpoint_id` equals `TARGET`. If attaching a separately recorded snapshot, assign its existing JSON path to `TARGET_CONTRACT_STATE` and append `--target-contract-state "$TARGET_CONTRACT_STATE"` to generation commands **before generating the set**. Do not add it to execution. Its fields are defined in `psy_node_common/src/rollback/plan.rs:72-90`; filtering occurs at `psy_node_common/src/rollback/generator.rs:847`. An omitted or mismatched snapshot does not block local rollback; **L1 force-state and relayer recovery are separate operator work**, never performed by this executor (`docs/src/dev/devnet_lifecycle.md:190,203`; `psy_cli/psy_dev_cli/src/subcommand/rollback.rs:148-159`). + +## 5. Inspect and validate the frozen RPs + +The files are mutable progress journals but their target, identity, snapshots, delete keys, and ordering are frozen. Never edit or reserialize an RP with `jq`: `proc_id` is an unsigned 128-bit integer, and JSON tooling can lose integer precision. Use `jq` read-only for inspection. The Rust reader enforces typed fields; the validator checks exact phase order, APIs, semantic keys, and marker-last ordering (`psy_node_common/src/rollback/plan.rs:63-70,108-131,157-203`; `psy_node_common/src/rollback/validate.rs:18-96`). + +```bash +PLAN_NAMES=(coordinator realm0sub1 realm0sub2 realm1sub1 realm1sub2) +check_plan() { + local name="$1" role="$2" realm="$3" sub="$4" status="$5" + jq -e --arg role "$role" --arg target "$TARGET" \ + --argjson realm "$realm" --argjson sub "$sub" --arg status "$status" ' + .role == $role and .realm_id == $realm and .realm_sub_id == $sub + and (.target_checkpoint_id | tostring) == $target + and (.latest_checkpoint_id >= .target_checkpoint_id) + and (.latest_pending_id | type == "number") + and (.ids | type == "array") + and (.snapshot.target_info | type == "string") + and (.snapshot.worker_reputation_fields | type == "array") + and (.phases | type == "array" and length > 1) + and all(.phases[]; + (.table | type == "string") and (.api | type == "string") + and (.keys | type == "array") and .status == $status) + and .phases[-2].table == "all" and .phases[-2].api == "verify" + and .phases[-1].table == "u64_singleton_table" + and .phases[-1].api == "set_latest_checkpoint_id" + and ((has("target_contract_state") | not) + or .target_contract_state.last_finalized_checkpoint_id == .target_checkpoint_id) + ' "$RP_DIR/$name.json" +} +check_all_plans() { + local status="$1" + check_plan coordinator coordinator 0 0 "$status" || return + check_plan realm0sub1 realm 0 1 "$status" || return + check_plan realm0sub2 realm 0 2 "$status" || return + check_plan realm1sub1 realm 1 1 "$status" || return + check_plan realm1sub2 realm 1 2 "$status" || return +} +check_all_plans pending +for name in "${PLAN_NAMES[@]}"; do + printf '\n%s\n' "$name" + jq '{role, realm_id, realm_sub_id, target_checkpoint_id, + latest_checkpoint_id, latest_pending_id, + phase_status: [.phases[] | {table, api, status}]}' "$RP_DIR/$name.json" +done +``` + +Require five readable files, the matrix identities, one common target, and every initial status `pending`. Compare each `latest_checkpoint_id` with the recorded stopped head, and retain each `latest_pending_id` as its frozen high-water; heads need not have been identical. The checks above cover the operator-visible schema and status, **not the complete Rust semantic validator**. Generation validates before persisting; execution validates again before mutating stores (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:135-138,150-154`; `psy_node_common/src/rollback/executor.rs:139-151,274-287`). + +## 6. Execute and require completion + +Run sequentially, stopping at the first error. Do not generate replacement RPs after any execution has started. + +```bash +rollback_coordinator --execute +rollback_realm 0 1 --execute +rollback_realm 0 2 --execute +rollback_realm 1 1 --execute +rollback_realm 1 2 --execute +check_all_plans completed +``` + +A zero exit code for one identity does not complete the other four. Require every phase in every RP to be `completed`, including empty-key phases, verification, and the final marker. The executor checks stored postconditions before the marker and preserves the exact pending high-water (`psy_node_common/src/rollback/plan.rs:107-115`; `psy_node_common/src/rollback/executor.rs:158-203,248-287`; `docs/src/dev/devnet_lifecycle.md:190`). + +## 7. Failure and resume procedure + +Throughout generation, execution, recovery, and resume: **no purge and no teardown**. This prohibits `make shutdown`, `PURGE=0 make shutdown`, `make restart-all`, launcher `--teardown`/`--purge`, Docker teardown, manual per-service restart, deleting retained files, and substituting `make run-all`. Preserve the original supervisor, Anvil and its snapshot/deployment pair, Scylla, Redis, NATS, checkpoint files, logs, keys, runtime config, sentinel, and RPs (`docs/src/dev/devnet_lifecycle.md:166,197,234-243`). + +| Failure | Required action | +|---|---| +| Missing/invalid sentinel or reachable endpoint | Retry `make rollback-stop` through the existing supervisor; resolve the tracked application stop failure. Never forge the sentinel or omit an endpoint (`docs/src/dev/devnet_lifecycle.md:226,230`). | +| Realm identity/config mismatch | Keep applications stopped. Check the matrix, namespace, `PSY_CONFIG_PATH`, and original identity-key path. Do not rotate keys or add `realm_sub_id` to the processor config (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:237-270`; `psy_node_core/src/config/node_cli_config.rs:34-52`). | +| Generation fails, target unavailable, or backup/history disagreement | Execute nothing until all five valid RPs exist. Diagnose retained state; never delete one side of a database/backup disagreement (`psy_cli/psy_dev_cli/src/subcommand/rollback/generate.rs:301-368`; `docs/src/dev/devnet_lifecycle.md:225`). | +| Execution phase or progress persistence fails | Keep all applications stopped, repair the reported retained-store or filesystem problem without clearing state, and rerun `--execute` against the same RP path. Never mark a phase complete by hand (`psy_node_common/src/rollback/executor.rs:158-198`). | +| Current marker differs from both frozen head and target, or pending counter differs | Stop recovery and investigate unexpected writes or the wrong config/RP. Do not alter markers, lower counters, or regenerate the RP to bypass the guard (`psy_node_common/src/rollback/executor.rs:142-151,274-287`). | +| Marker is already the target after an interrupted execution | Rerun the same RP. Reconciliation verifies postconditions and persists missing completion statuses; it does not authorize ignoring failures (`psy_node_common/src/rollback/executor.rs:153-155,206-245`). | +| Missing/corrupt RP after destructive work | Keep applications stopped; recover the original frozen RP from a trusted preserved copy. Do not regenerate it from partially rolled-back stores (`docs/src/dev/devnet_lifecycle.md:242`). | +| Resume fails | Preserve the sentinel and infrastructure, fix the failing application, then rerun `make rollback-resume`. The supervisor stops the newly started subset on failure (`docs/src/dev/devnet_lifecycle.md:232`; `docs/src/dev/devnet-launcher-reference.md:554-558`). | + +For example, if `realm0sub2` failed, retain the existing shell variables/functions and run: + +```bash +rollback_realm 0 2 --execute +rollback_realm 1 1 --execute +rollback_realm 1 2 --execute +check_all_plans completed +``` + +If the operator shell was lost, return to the same repository root, restore `WORKSPACE` to the existing directory, read `TARGET` from `"$WORKSPACE/target-checkpoint.txt"`, and restore the assignments/functions from sections 2, 4, and 5. **Do not rerun workspace creation, config generation, RP generation, or the initial `pending` gate.** Reexecuting already-completed RPs while still offline invokes reconciliation (`psy_node_common/src/rollback/executor.rs:153-155`). + +## 8. Resume and post-E2E acceptance + +Only after `check_all_plans completed` succeeds and the separately authorized L1/relayer recovery is verified complete: + +```bash +make rollback-resume +test ! -e "$STOP_SENTINEL" +``` + +This restores saved application templates; it does not redeploy L1 or reset Envio. The sentinel is removed after startup conditions succeed. Plonky2 worker/proxy initialization can take minutes; ongoing controlled-start markers and progressively opening readiness ports are progress, not permission to replace the supervisor (`docs/src/dev/devnet_lifecycle.md:191-199`; `dev/locSetupV4.ts:4019-4039`). + +Apply the complete [post-rollback acceptance checklist](devnet_lifecycle.md#7-post-restart-and-post-rollback-verification), not just an open port or an accepted submission (`docs/src/dev/devnet_lifecycle.md:201-210`): + +- Require preserved L1 continuity, byte-identical deployed addresses, and a valid Anvil snapshot. Record explicitly that these commands attached no target contract snapshot; independently verify external recovery. +- Require the exact Coordinator readiness marker `[COORD_CREATE] processor new done` and each Realm marker `[REALM_CREATE] processor new done`, then all five heads above `TARGET` and converged (`docs/src/dev/devnet-launcher-reference.md:410-413`). +- Require target checkpoint queryability and restored application state. Submit the original pre-rollback state-changing transaction when repeatable, require committed confirmation, and observe continued head advancement. +- Require contiguous checkpoint ring buffers and new gatherer backups above each RP's retained pending high-water. Historical end-cap checkpoint proofs must be bounded by the current logical head. + +Keep the five completed RPs and before/after evidence together. A completed local RP is not proof of L1 recovery, and transaction admission alone is not E2E success (`docs/src/dev/devnet_lifecycle.md:203-212`). + +## 9. Security + +- Treat rollback as destructive privileged maintenance. Restrict workspace access (`umask 077` above), retain original evidence, and do not expose private keys or credentials in tickets, logs, or version control. +- Use key **paths**, never inline key contents. Keep the runtime public validator registry and the original identity keys unchanged; identity matching is a safety check, not an obstacle to bypass (`psy_cli/psy_node_cli/src/node/realm_p2p.rs:359-382`). +- Do not hand-edit delete keys, phases, statuses, snapshots, counters, or markers. Typed decoding and semantic validation are deliberately fail-closed (`psy_node_common/src/rollback/plan.rs:63-131`; `psy_node_common/src/rollback/validate.rs:18-96`). +- Use only `plonky2-poseidon-goldilocks`. JTMB is not rollback evidence and is rejected by this command (`psy_cli/psy_dev_cli/src/subcommand/rollback.rs:173-177`; `AGENTS.md:39-41`). diff --git a/docs/src/node/getting_started.md b/docs/src/node/getting_started.md index 4e9e5067..36596cd6 100644 --- a/docs/src/node/getting_started.md +++ b/docs/src/node/getting_started.md @@ -1,6 +1,6 @@ # Getting Started -> For supervised start/stop/restart/rollback, follow the internal guides `docs/src/../dev/devnet_lifecycle.md` and `docs/src/../dev/devnet-launcher-reference.md` (not published in SUMMARY). Prefer `make run-all` / `make shutdown` over manual per-service starts. +> For supervised start/stop/restart/rollback, follow the internal guides `docs/src/../dev/devnet_lifecycle.md` and `docs/src/../dev/devnet-launcher-reference.md` (not published in SUMMARY). Prefer `make run-all` / `make shutdown` over manual per-service starts. Note that `make shutdown` purges persisted chain state by default; use `PURGE=0 make shutdown` to stop the stack while preserving it. > Updated: 2026-09-03. @@ -40,7 +40,7 @@ Run the supported launcher from ``: make run-all ``` -The launcher starts the configured database, coordinator, realms, workers, proving services, layer-one services, relayer, and selected application surfaces. The target is defined in `Makefile:60-66`. +The launcher starts the configured database, coordinator, realms, workers, proving services, layer-one services, relayer, and selected application surfaces. The target is defined in `Makefile:61-67`. ## 3. Required Components @@ -246,10 +246,10 @@ HTTP admission alone is not end-to-end acceptance. Follow the lifecycle guide fo Stop the supervised stack without deleting state: ```bash -make shutdown +PURGE=0 make shutdown ``` -The shutdown target invokes the launcher teardown path (`Makefile:102-103`). Do not remove data directories manually. +`make shutdown` purges by default (`PURGE ?= 1` in the Makefile): it deletes checkpoints, `db/anvil/state.json`, logs, localhost deployments, and devnet Docker volumes; `PURGE=1 make shutdown` is the explicit equivalent. The shutdown target invokes the launcher teardown path (`Makefile:103-104`). Do not remove data directories manually. ## 9. Implemented Network Capabilities diff --git a/parth_common/src/tree_sync/traits.rs b/parth_common/src/tree_sync/traits.rs index c87f4472..cc15a798 100644 --- a/parth_common/src/tree_sync/traits.rs +++ b/parth_common/src/tree_sync/traits.rs @@ -749,7 +749,7 @@ fn perform_smart_rehash>( +pub fn rehash_sparse_paths>( local_tree: &mut Destination, nodes: &[SimpleMerkleNodeKey], target_level: u8, diff --git a/psy_api_core/src/coordinator/standard_edge_rpc.rs b/psy_api_core/src/coordinator/standard_edge_rpc.rs index 669220a1..74658604 100644 --- a/psy_api_core/src/coordinator/standard_edge_rpc.rs +++ b/psy_api_core/src/coordinator/standard_edge_rpc.rs @@ -40,7 +40,7 @@ pub trait CoordinatorEdgeRpc: NodeEdgeWorkerRpcServer RpcResult; #[method(name = "submit_guta")] - async fn submit_guta(&self, input: GlobalUserTreeAggregatorHeaderWithTagValueAndJobType, proof: Vec, realm_id: u64, proposal: Option>, certificate: Option>, finalize_binding: Vec) -> RpcResult; + async fn submit_guta(&self, input: GlobalUserTreeAggregatorHeaderWithTagValueAndJobType, proof: Vec, realm_id: u64, proposal: Option>, certificate: Option>) -> RpcResult; #[method(name = "get_latest_checkpoint_id")] diff --git a/psy_cli/psy_node_cli/src/node/realm_p2p.rs b/psy_cli/psy_node_cli/src/node/realm_p2p.rs index 716672a6..1e832643 100644 --- a/psy_cli/psy_node_cli/src/node/realm_p2p.rs +++ b/psy_cli/psy_node_cli/src/node/realm_p2p.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use parth_core::{ @@ -15,7 +16,7 @@ use psy_data::{ genesis::genesis_block_setup::PsyGenesisBlockSetupData, p2p::{ BlsPublicKey, EndCapForwardHeader, EndCapForwardResponse, EndCapRejectReason, NodeId, - MAX_VALIDATORS_PER_REALM, MIN_VALIDATORS_PER_REALM, + ProposalLookupResponse, ValidatorLeaf, MAX_VALIDATORS_PER_REALM, MIN_VALIDATORS_PER_REALM, }, }; use psy_node_common::{ @@ -23,10 +24,14 @@ use psy_node_common::{ realm::{ network::{ build_optional_realm_network, load_bls_secret_key, load_ed25519_identity_key, - parse_bootnode, run_realm_network, ProposalWithBody, OptionalRealmNetwork, + parse_bootnode, run_realm_network, OptionalRealmNetwork, RealmNetworkCommands, RealmNetworkEvent, }, - processor::consensus::{sign_vote, verify_proposal_submission}, + processor::{ + consensus::{decode_proposal_state_updates, sign_vote, verify_proposal_submission}, + proposal_store::ProposalStore, + recovery::BaselineReplayRequest, + }, }, }; use psy_node_core::{ @@ -492,40 +497,64 @@ pub fn validate_processor_identity( Ok(derived_sub_id) } -/// Drive loop plus processor event consumer. Non-proposers validate, deliver the -/// complete proposal, and vote. +pub async fn processor_realm_validator_leaves( + store: &S, + chain_id: u64, + realm_id: u32, +) -> anyhow::Result> +where + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Sync, + N::HasherBase: MerkleZeroHasher, + N::QHash: Copy + PartialEq + Q256BitHash, +{ + let checkpoint_id = store.get_latest_checkpoint_id().await?; + let roots = store.get_checkpoint_global_state_roots(checkpoint_id).await?; + let (_, _, _, leaves) = load_realm_validators_from_tree::( + store, + chain_id, + checkpoint_id, + realm_id, + &roots.validator_tree_root, + ) + .await?; + Ok(leaves.into_iter().map(|(_, leaf)| leaf).collect()) +} + +/// Drive loop plus processor event consumer. Lookup/range serve immediately. +/// Follower Vote waits until production runtime enables voting, then stores the +/// proposal and requires baseline replay before sign/publish. pub fn spawn_processor_realm_network( - built: OptionalRealmNetwork, + mut built: OptionalRealmNetwork, config: &RealmProcessorStartConfig, local_sub_id: u16, validator_store: Arc, - proof_verifier: N::ZKVerifier, - proposal_tx: tokio::sync::mpsc::Sender, + proof_verifier: Arc, + proposal_store: Arc, + validator_leaves: Vec, + commands: RealmNetworkCommands, + bls_secret: psy_data::p2p::BlsSecretKey, + vote_enabled: Arc, + baseline_replay: tokio::sync::mpsc::Sender>, ) where N: QNetworkTypesConfig + 'static, S: PsyRealmProcessorStore + Send + Sync + 'static, N::HasherBase: MerkleZeroHasher, - N::ZKVerifier: 'static, + N::ZKVerifier: Send + Sync + 'static, + N::QHash: Q256BitHash + Send + 'static, { + built.network.set_validator_leaves(validator_leaves); let OptionalRealmNetwork { network, handle, - bls_secret, .. } = built; let realm_id = config.realm_id as u32; let chain_id = config.network.get_chain_id(); - let public = realm_public_data(config.network, realm_id) - .expect("processor Realm P2P public config was validated at startup"); - let proof_verifier = Arc::new(proof_verifier); - let commands = handle.commands(); + let bls_secret = Arc::new(bls_secret); let mut events = handle.into_parts().1; tokio::spawn(run_realm_network(network)); tokio::spawn(async move { - let Some(bls_secret) = bls_secret else { - tracing::error!("processor P2P event loop missing BLS secret"); - return; - }; while let Some(event) = events.recv().await { match event { RealmNetworkEvent::ProposalReady { source, proposal, body } => { @@ -533,6 +562,14 @@ pub fn spawn_processor_realm_network( continue; } let proposal_id = proposal.proposal_id; + let validator_store = validator_store.clone(); + let proof_verifier = proof_verifier.clone(); + let proposal_store = proposal_store.clone(); + let vote_enabled = vote_enabled.clone(); + let baseline_replay = baseline_replay.clone(); + let commands = commands.clone(); + let bls_secret = bls_secret.clone(); + tokio::spawn(async move { let validation = async { anyhow::ensure!(proposal.chain_id == chain_id, "Proposal chain_id mismatch"); anyhow::ensure!(proposal.realm_id == realm_id, "Proposal realm_id mismatch"); @@ -540,14 +577,10 @@ pub fn spawn_processor_realm_network( proposal.compute_proposal_id() == proposal.proposal_id, "Proposal proposal_id mismatch" ); - anyhow::ensure!( - public.validator_processor_node_ids.get(&proposal.proposer_sub_id) == Some(&source), - "Proposal source NodeId does not match configured proposer" - ); let roots = validator_store .get_checkpoint_global_state_roots(proposal.base_checkpoint_id) .await?; - let (_, _, user_ids) = load_realm_validators_from_tree::( + let (_, _, user_ids, leaves) = load_realm_validators_from_tree::( validator_store.as_ref(), chain_id, proposal.base_checkpoint_id, @@ -555,6 +588,14 @@ pub fn spawn_processor_realm_network( &roots.validator_tree_root, ) .await?; + let expected_source = leaves + .iter() + .find(|(sub_id, _)| *sub_id == proposal.proposer_sub_id) + .map(|(_, leaf)| leaf.node_id); + anyhow::ensure!( + expected_source == Some(source), + "Proposal source NodeId does not match validator leaf at proof-base checkpoint" + ); anyhow::ensure!( proposal.validator_tree_root == roots.validator_tree_root.into_owned_32bytes(), "Proposal validator_tree_root does not match proof-base checkpoint" @@ -566,46 +607,72 @@ pub fn spawn_processor_realm_network( "GUTA proposer sub_id {} has no checkpoint validator", proposal.proposer_sub_id ))?; - verify_proposal_submission::( + let decoded = verify_proposal_submission::( &proposal, body.as_bytes(), proposer_user_id, proof_verifier.as_ref(), )?; + proposal_store.save_proposal(&proposal, body.as_bytes()).await?; + if !vote_enabled.load(Ordering::Acquire) { + return Ok(()); + } + let updates = decode_proposal_state_updates::(&decoded.state_updates)?; + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + baseline_replay + .send(BaselineReplayRequest { + previous_checkpoint_id: proposal.base_checkpoint_id, + updates, + reply: reply_tx, + }) + .await + .map_err(|_| anyhow::anyhow!("baseline replay receiver dropped"))?; + match reply_rx.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!( + "follower baseline replay rejected proposal={} error={error:#}", + hex::encode(proposal.proposal_id) + ); + return Ok(()); + } + Err(_) => anyhow::bail!("baseline replay reply dropped"), + } let vote = sign_vote(&bls_secret, local_sub_id, &proposal); - let complete = ProposalWithBody { proposal, body }; - proposal_tx - .send(complete) + commands + .publish_vote(vote) .await - .map_err(|_| anyhow::anyhow!("verified proposals receiver dropped"))?; - Ok::<_, anyhow::Error>(vote) + .map_err(|error| anyhow::anyhow!("follower vote publish failed: {error}"))?; + Ok::<_, anyhow::Error>(()) }.await; - let vote = match validation { - Ok(vote) => vote, - Err(error) => { - tracing::warn!( - "realm P2P non-proposer rejected Proposal proposal={} error={:#}", - hex::encode(proposal_id), - error - ); - continue; - } - }; - if let Err(error) = commands.publish_vote(vote).await { + if let Err(error) = validation { tracing::warn!( - "realm P2P non-proposer vote publish failed proposal={} error={}", + "realm P2P non-proposer rejected Proposal proposal={} error={:#}", hex::encode(proposal_id), error ); - continue; } - tracing::info!( - "realm P2P non-proposer vote published proposal={} signer_sub_id={} realm={} source={:?}", - hex::encode(proposal_id), - local_sub_id, - realm_id, - source - ); + }); + } + RealmNetworkEvent::LookupReceived { request, reply, .. } => { + let response = match proposal_store.lookup_proposal(&request).await { + Ok(response) => response, + Err(error) => { + tracing::warn!("ProposalLookup serve failed error={error:#}"); + ProposalLookupResponse::empty() + } + }; + let _ = reply.send(response); + } + RealmNetworkEvent::DirectBodyReceived { request, reply, .. } => { + match proposal_store.read_body_chunk(&request).await { + Ok(response) => { + let _ = reply.send(response); + } + Err(error) => { + tracing::warn!("direct range serve failed error={error:#}"); + } + } } RealmNetworkEvent::EndCapReceived { reply, .. } => { let _ = reply.send(EndCapForwardResponse::rejected(EndCapRejectReason::Invalid)); @@ -645,6 +712,8 @@ where let _ = reply.send(response); } RealmNetworkEvent::ProposalReady { .. } + | RealmNetworkEvent::LookupReceived { .. } + | RealmNetworkEvent::DirectBodyReceived { .. } | RealmNetworkEvent::VoteReceived { .. } => {} } } diff --git a/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs b/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs index a94cb428..5d0f6d5f 100644 --- a/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs +++ b/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; @@ -297,7 +298,7 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce let config = &config.clone().with_derived_realm_sub_id(realm_sub_id); let (circuit_library, circuit_manager) = psy_plonky2_circuits::circuit_library::get_plonky2_circuit_library_and_prover_for_network::(config.network)?; drop(circuit_manager); - let verifier = PsyPlonky2ZKVerifier::::new(circuit_library); + let proof_verifier = Arc::new(PsyPlonky2ZKVerifier::::new(circuit_library)); let pool = new_redis_async_pool(&config.redis_url, 2).await?; @@ -363,7 +364,47 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce let coordinator_client = PsyRealmCoordinatorClientAPI::::new( http_client, ); - let (mut processor, guta_gatherer_join_handle) = create_realm_processor::( + let proposal_store = Arc::new( + psy_node_common::realm::processor::proposal_store::ProposalStore::open( + config.get_proposal_backups_path(), + ) + .await?, + ); + let built = crate::node::realm_p2p::build_processor_network(config, chain_id)?; + let validator_leaves = match crate::node::realm_p2p::processor_realm_validator_leaves::( + db.as_ref(), + chain_id, + config.realm_id as u32, + ) + .await + { + Ok(leaves) => leaves, + Err(error) => { + tracing::warn!("checkpoint validator leaves not loaded before genesis: {error:#}"); + Vec::new() + } + }; + let commands = built.handle.commands(); + let rotation = built.rotation.clone(); + let bls_key_path = config.p2p_bls_key_path.as_deref().ok_or_else(|| { + anyhow::anyhow!("processor P2P requires --p2p-bls-key") + })?; + let vote_enabled = Arc::new(AtomicBool::new(false)); + let (replay_tx, replay_rx) = tokio::sync::mpsc::channel(8); + crate::node::realm_p2p::spawn_processor_realm_network::( + built, + config, + realm_sub_id, + validator_store, + proof_verifier.clone(), + proposal_store.clone(), + validator_leaves, + commands.clone(), + load_bls_secret_key(bls_key_path)?, + vote_enabled.clone(), + replay_tx, + ); + let mut processor = create_realm_processor::( chain_id, &genesis_data, file_system, @@ -378,31 +419,20 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce realm_identifier, circuit_fingerprint_config, Arc::new(coordinator_client), + proof_verifier, + proposal_store, + Some(commands.clone()), ) .await?; - let built = crate::node::realm_p2p::build_processor_network(config, chain_id)?; - let bls_secret = load_bls_secret_key(config.p2p_bls_key_path.as_deref().ok_or_else(|| { - anyhow::anyhow!("processor P2P requires --p2p-bls-key") - })?)?; - let commands = built.handle.commands(); - let rotation = built.rotation.clone(); - processor.set_realm_p2p(commands, rotation, bls_secret); - let (proposal_tx, proposal_rx) = tokio::sync::mpsc::channel(4); - processor.proposal_rx = Some(proposal_rx); - crate::node::realm_p2p::spawn_processor_realm_network::( - built, - config, - realm_sub_id, - validator_store, - verifier, - proposal_tx, - ); + processor.set_baseline_replay_rx(replay_rx); + processor.set_realm_p2p(commands, rotation, load_bls_secret_key(bls_key_path)?); + vote_enabled.store(true, Ordering::Release); - run_realm_processor(processor, guta_gatherer_join_handle).await?; + run_realm_processor(processor).await?; } _ => { anyhow::bail!("Unsupported network type '{:?}' for Plonky2 Scylla coordinator processor node", config.network ); diff --git a/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs b/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs index 17dd806f..749ce3fb 100644 --- a/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs +++ b/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; @@ -107,6 +108,7 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_coordinator_processor_n pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(config: &RealmProcessorStartConfig) -> anyhow::Result<()> { let (verifier, _) = get_jtmb_circuit_library_and_prover_for_network::(config.network)?; + let proof_verifier = Arc::new(PsyJTMBZKVerifier::new(verifier)); let resolver = PsyJTMBPoseidonGoldilocksNodeConfigResolver {}; let circuit_fingerprint_config = resolver.get_circuit_fingerprint_config_for_network(config.network)?; let genesis_data = resolver.get_genesis_block_setup_data_for_network(config.network, config.genesis_data_path.clone())?; @@ -161,7 +163,47 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(co http_client, ); tracing::info!("[REALM_BOOT] creating realm processor"); - let (mut processor, guta_gatherer_join_handle) = create_realm_processor::( + let proposal_store = Arc::new( + psy_node_common::realm::processor::proposal_store::ProposalStore::open( + config.get_proposal_backups_path(), + ) + .await?, + ); + let built = crate::node::realm_p2p::build_processor_network(config, chain_id)?; + let validator_leaves = match crate::node::realm_p2p::processor_realm_validator_leaves::( + db.as_ref(), + chain_id, + config.realm_id as u32, + ) + .await + { + Ok(leaves) => leaves, + Err(error) => { + tracing::warn!("checkpoint validator leaves not loaded before genesis: {error:#}"); + Vec::new() + } + }; + let commands = built.handle.commands(); + let rotation = built.rotation.clone(); + let bls_key_path = config.p2p_bls_key_path.as_deref().ok_or_else(|| { + anyhow::anyhow!("processor P2P requires --p2p-bls-key") + })?; + let vote_enabled = Arc::new(AtomicBool::new(false)); + let (replay_tx, replay_rx) = tokio::sync::mpsc::channel(8); + crate::node::realm_p2p::spawn_processor_realm_network::( + built, + config, + realm_sub_id, + validator_store, + proof_verifier.clone(), + proposal_store.clone(), + validator_leaves, + commands.clone(), + load_bls_secret_key(bls_key_path)?, + vote_enabled.clone(), + replay_tx, + ); + let mut processor = create_realm_processor::( chain_id, &genesis_data, file_system, @@ -176,31 +218,15 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(co realm_identifier, circuit_fingerprint_config, Arc::new(coordinator_client), + proof_verifier, + proposal_store, + Some(commands.clone()), ) .await?; - let built = crate::node::realm_p2p::build_processor_network(config, chain_id)?; - let bls_secret = load_bls_secret_key(config.p2p_bls_key_path.as_deref().ok_or_else(|| { - anyhow::anyhow!("processor P2P requires --p2p-bls-key") - })?)?; - let commands = built.handle.commands(); - let rotation = built.rotation.clone(); - processor.set_realm_p2p(commands, rotation, bls_secret); - let (proposal_tx, proposal_rx) = tokio::sync::mpsc::channel(4); - processor.proposal_rx = Some(proposal_rx); - crate::node::realm_p2p::spawn_processor_realm_network::( - built, - config, - realm_sub_id, - validator_store, - PsyJTMBZKVerifier::new(verifier), - proposal_tx, - ); - - - - - - run_realm_processor(processor, guta_gatherer_join_handle).await?; + processor.set_baseline_replay_rx(replay_rx); + processor.set_realm_p2p(commands, rotation, load_bls_secret_key(bls_key_path)?); + vote_enabled.store(true, Ordering::Release); + run_realm_processor(processor).await?; tracing::info!("[REALM_BOOT] realm processor exited"); } _ => { diff --git a/psy_data/bindings/PsyRealmUserUpdateQueueItem.ts b/psy_data/bindings/PsyRealmUserUpdateQueueItem.ts index ec104712..632dce40 100644 --- a/psy_data/bindings/PsyRealmUserUpdateQueueItem.ts +++ b/psy_data/bindings/PsyRealmUserUpdateQueueItem.ts @@ -5,4 +5,4 @@ import type { QEDUserLeaf } from "./QEDUserLeaf"; import type { QHashOut } from "./QHashOut"; import type { QProvingJobDataID } from "./QProvingJobDataID"; -export type PsyRealmUserUpdateQueueItem = { job_id: QProvingJobDataID, expected_fake_checkpoint_id: bigint, old_user_leaf_hash: QHashOut, new_user_leaf_hash: QHashOut, new_user_leaf: QEDUserLeaf, stats: GUTAStats, events: Array, }; +export type PsyRealmUserUpdateQueueItem = { job_id: QProvingJobDataID, submission_nonce: bigint, old_user_leaf_hash: QHashOut, new_user_leaf_hash: QHashOut, new_user_leaf: QEDUserLeaf, stats: GUTAStats, events: Array, }; diff --git a/psy_data/src/guta/realm_finalize.rs b/psy_data/src/guta/realm_finalize.rs index d9ac89b5..40042029 100644 --- a/psy_data/src/guta/realm_finalize.rs +++ b/psy_data/src/guta/realm_finalize.rs @@ -16,7 +16,7 @@ use crate::{ stats::GUTAStats, sub_tree_transition::SubTreeNodeStateTransition, }, - p2p::{validate_goldilocks_limb, validate_hash32_canonical, DOMAIN_VALIDATOR_LEAF_FELT, ProtocolError, ProtocolReader, ProtocolResult, write_fixed, write_u16, write_u64}, + p2p::{validate_goldilocks_limb, DOMAIN_VALIDATOR_LEAF_FELT, ProtocolError, ProtocolReader, ProtocolResult, write_fixed, write_u16, write_u64}, v1::qdata::{ checkpoint::{PQEDCheckpointLeaf, PQEDCheckpointLeafCompactWithStateRoots}, user::PQEDUserLeaf, @@ -283,60 +283,6 @@ where }) } -// ================================================================================= -// Finalizer Binding (off-circuit BLS authorization material) -// ================================================================================= - -/// Exact wire length of [`RealmFinalizeBinding`]: 410-byte output + 32-byte tag. -pub const REALM_FINALIZE_BINDING_WIRE_BYTES: usize = 410 + 32; - -/// Public binding payload required by the `psy_submit_guta` Realm admission gate. -/// -/// It carries the actual canonical circuit output plus the finalizer worker -/// reward tag needed to recompute the circuit reward root. It contains no -/// signature and no secret: authorization is the Coordinator-verified BLS -/// certificate over the proposal identity derived from this output. -/// -/// Wire encoding is direct concatenation in declaration order, exactly -/// [`REALM_FINALIZE_BINDING_WIRE_BYTES`]; decoders reject trailing bytes. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RealmFinalizeBinding { - /// Actual canonical finalizer output; required, no default. - pub output: [u8; 410], - /// Canonical field-hash bytes of the finalizer worker reward tag. - pub finalizer_worker_reward_tag: [u8; 32], -} - -impl RealmFinalizeBinding { - pub fn protocol_encode_to_vec(&self) -> Vec { - let mut out = Vec::with_capacity(REALM_FINALIZE_BINDING_WIRE_BYTES); - write_fixed(&mut out, &self.output); - write_fixed(&mut out, &self.finalizer_worker_reward_tag); - out - } - - /// Strictly decode a 442-byte binding; rejects wrong length and - /// noncanonical worker-tag field limbs. - pub fn protocol_decode(bytes: &[u8]) -> ProtocolResult { - if bytes.len() != REALM_FINALIZE_BINDING_WIRE_BYTES { - return Err(ProtocolError::InvalidLength { - what: "RealmFinalizeBinding", - got: bytes.len(), - expected: REALM_FINALIZE_BINDING_WIRE_BYTES, - }); - } - let mut output = [0u8; 410]; - output.copy_from_slice(&bytes[..410]); - let mut tag = [0u8; 32]; - tag.copy_from_slice(&bytes[410..]); - validate_hash32_canonical(&tag)?; - Ok(Self { - output, - finalizer_worker_reward_tag: tag, - }) - } -} - /// Reconstruct the actual finalizer output from the exact planner witness /// artifacts and the root GUTA reward tag. /// @@ -622,62 +568,6 @@ mod chain_domain_tests { } } -#[cfg(test)] -mod finalize_binding_tests { - use super::*; - use crate::p2p::{GOLDILOCKS_MODULUS, ProtocolError}; - - #[test] - fn binding_roundtrip_accepts_largest_canonical_tag() { - let mut tag = [0u8; 32]; - tag[24..32].copy_from_slice(&(GOLDILOCKS_MODULUS - 1).to_le_bytes()); - let binding = RealmFinalizeBinding { - output: [0x11; 410], - finalizer_worker_reward_tag: tag, - }; - let encoded = binding.protocol_encode_to_vec(); - assert_eq!(encoded.len(), REALM_FINALIZE_BINDING_WIRE_BYTES); - assert_eq!(RealmFinalizeBinding::protocol_decode(&encoded).unwrap(), binding); - } - - #[test] - fn binding_decode_rejects_noncanonical_tag_and_wrong_length() { - let mut tag = [0u8; 32]; - tag[24..32].copy_from_slice(&GOLDILOCKS_MODULUS.to_le_bytes()); - let binding = RealmFinalizeBinding { - output: [0x22; 410], - finalizer_worker_reward_tag: tag, - }; - let encoded = binding.protocol_encode_to_vec(); - assert!(matches!( - RealmFinalizeBinding::protocol_decode(&encoded), - Err(ProtocolError::NonCanonicalField { .. }) - )); - - let ok_tag = [0u8; 32]; - let mut truncated = RealmFinalizeBinding { - output: [0x22; 410], - finalizer_worker_reward_tag: ok_tag, - } - .protocol_encode_to_vec(); - truncated.pop(); - assert!(matches!( - RealmFinalizeBinding::protocol_decode(&truncated), - Err(ProtocolError::InvalidLength { .. }) - )); - let mut trailing = RealmFinalizeBinding { - output: [0x22; 410], - finalizer_worker_reward_tag: ok_tag, - } - .protocol_encode_to_vec(); - trailing.push(0); - assert!(matches!( - RealmFinalizeBinding::protocol_decode(&truncated), - Err(ProtocolError::InvalidLength { .. }) - )); - } -} - #[cfg(test)] mod finalize_reward_root63_tests { use super::*; diff --git a/psy_data/src/p2p/limits.rs b/psy_data/src/p2p/limits.rs index 3425f529..651d16af 100644 --- a/psy_data/src/p2p/limits.rs +++ b/psy_data/src/p2p/limits.rs @@ -26,7 +26,7 @@ pub const MAX_PROPOSAL_PARTS: u32 = MAX_PROPOSAL_BODY_BYTES.div_ceil(MAX_PROPOSAL_CHUNK_BYTES) as u32; /// Maximum direct body-range response payload. -pub const DIRECT_REQUEST_MAX_BYTES: u32 = 61_440; +pub const BODY_CHUNK_MAX_BYTES: u32 = 61_440; /// Maximum EndCap forward stream total (header + input + proof). pub const MAX_END_CAP_FORWARD_BYTES: usize = 536_870_912; @@ -40,8 +40,8 @@ pub const VOTE_WIRE_BYTES: usize = 130; /// Fixed Certificate wire length. pub const CERTIFICATE_WIRE_BYTES: usize = 204; -/// Fixed DirectBodyRequest wire length. -pub const DIRECT_BODY_REQUEST_WIRE_BYTES: usize = 44; +/// Fixed BodyChunkRequest wire length. +pub const BODY_CHUNK_REQUEST_WIRE_BYTES: usize = 44; /// EndCapForwardHeader wire length (60 bytes): /// `chain_id(8) + realm_id(4) + checkpoint_id(8) + end_cap_id(32) @@ -73,12 +73,33 @@ pub const MAX_VALIDATORS_PER_REALM: usize = 64; /// Minimum occupied validators per Realm. pub const MIN_VALIDATORS_PER_REALM: usize = 1; -/// Maximum checkpoints between a proposal's proof base and coordinator inclusion. -pub const MAX_INCLUSION_LAG_CHECKPOINTS: u64 = 16; +/// Maximum windowed ProposalLookup response bytes. +pub const MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES: usize = 16_384; + +/// Concurrent ProposalLookup/direct-range peers on the recovery client. +pub const PROPOSAL_LOOKUP_CONCURRENCY: usize = 4; + +/// Per-round ProposalLookup/direct-range wait (seconds). +pub const PROPOSAL_LOOKUP_ROUND_SECS: u64 = 5; + +/// Total ProposalLookup/direct-range client budget (seconds). +pub const PROPOSAL_LOOKUP_TIMEOUT_SECS: u64 = 30; /// Maximum concurrent direct body exchanges. pub const MAX_CONCURRENT_DIRECT_EXCHANGES: usize = 64; +/// Concurrent in-flight proposal reassemblies and recorded Start sources. +pub const MAX_IN_FLIGHT_PROPOSALS: usize = 2; + +/// Vote-auth / backlog entries kept without an active waiter. +pub const MAX_VOTE_AUTH: usize = MAX_IN_FLIGHT_PROPOSALS; + +/// Vote-auth TTL when no waiter is attached (seconds). +pub const VOTE_AUTH_TTL_SECS: u64 = 1_800; + +/// Proposal reassembly expiry (seconds). +pub const REASSEMBLY_EXPIRY_SECS: u64 = 1_800; + /// Direct request exchange timeout (seconds). pub const DIRECT_REQUEST_TIMEOUT_SECS: u64 = 30; @@ -110,6 +131,14 @@ mod tests { assert_eq!(VOTE_WIRE_BYTES, 130); assert_eq!(CERTIFICATE_WIRE_BYTES, 204); assert_eq!(END_CAP_FORWARD_HEADER_WIRE_BYTES, 60); + assert_eq!(MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, 16_384); + assert_eq!(PROPOSAL_LOOKUP_CONCURRENCY, 4); + assert_eq!(PROPOSAL_LOOKUP_ROUND_SECS, 5); + assert_eq!(PROPOSAL_LOOKUP_TIMEOUT_SECS, 30); + assert_eq!(MAX_IN_FLIGHT_PROPOSALS, 2); + assert_eq!(MAX_VOTE_AUTH, 2); + assert_eq!(VOTE_AUTH_TTL_SECS, 1_800); + assert_eq!(REASSEMBLY_EXPIRY_SECS, 1_800); assert_eq!(replication_threshold(1), 1); assert_eq!(replication_threshold(2), 1); assert_eq!(replication_threshold(3), 2); diff --git a/psy_data/src/p2p/messages.rs b/psy_data/src/p2p/messages.rs index 748bf433..c8c79070 100644 --- a/psy_data/src/p2p/messages.rs +++ b/psy_data/src/p2p/messages.rs @@ -16,11 +16,11 @@ use super::codec::{ use super::domains::{DOMAIN_END_CAP_FORWARD, DOMAIN_PROPOSAL, DOMAIN_VOTE}; use super::error::{ProtocolError, ProtocolResult}; use super::limits::{ - CERTIFICATE_WIRE_BYTES, DIRECT_BODY_REQUEST_WIRE_BYTES, DIRECT_REQUEST_MAX_BYTES, + CERTIFICATE_WIRE_BYTES, BODY_CHUNK_REQUEST_WIRE_BYTES, BODY_CHUNK_MAX_BYTES, END_CAP_FORWARD_HEADER_WIRE_BYTES, END_CAP_FORWARD_RESPONSE_WIRE_BYTES, MAX_BACKUP_BYTES, MAX_FINALIZER_OUTPUT_BYTES, MAX_FINALIZER_PROOF_BYTES, MAX_PROPOSAL_BODY_BYTES, - MAX_PROPOSAL_CHUNK_BYTES, MAX_PROPOSAL_PARTS, PROPOSAL_WIRE_BYTES, - VOTE_WIRE_BYTES, + MAX_PROPOSAL_CHUNK_BYTES, MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, MAX_PROPOSAL_PARTS, + PROPOSAL_WIRE_BYTES, VOTE_WIRE_BYTES, }; /// Canonical fixed-size Realm finalizer public output (exactly 410 bytes). @@ -606,25 +606,25 @@ pub fn bitmap_set(bitmap: &mut [u8; 32], sub_id: u16) { /// Direct proposal-body range request (exactly 44 bytes). #[derive(Clone, Debug, Eq, PartialEq)] -pub struct DirectBodyRequest { +pub struct BodyChunkRequest { pub proposal_id: [u8; 32], pub offset: u64, pub max_bytes: u32, } -impl DirectBodyRequest { +impl BodyChunkRequest { /// Exact wire length (44 bytes). - pub const WIRE_BYTES: usize = DIRECT_BODY_REQUEST_WIRE_BYTES; + pub const WIRE_BYTES: usize = BODY_CHUNK_REQUEST_WIRE_BYTES; pub fn protocol_decode(reader: &mut ProtocolReader<'_>) -> ProtocolResult { let proposal_id = reader.read_bytes_32()?; let offset = reader.read_u64()?; let max_bytes = reader.read_u32()?; - if max_bytes == 0 || max_bytes > DIRECT_REQUEST_MAX_BYTES { + if max_bytes == 0 || max_bytes > BODY_CHUNK_MAX_BYTES { return Err(ProtocolError::LengthLimit { - what: "DirectBodyRequest.max_bytes", + what: "BodyChunkRequest.max_bytes", got: max_bytes as u64, - max: DIRECT_REQUEST_MAX_BYTES as u64, + max: BODY_CHUNK_MAX_BYTES as u64, }); } Ok(Self { @@ -639,7 +639,7 @@ impl DirectBodyRequest { } } -impl ProtocolEncode for DirectBodyRequest { +impl ProtocolEncode for BodyChunkRequest { fn protocol_encode(&self, out: &mut Vec) { write_fixed(out, &self.proposal_id); write_u64(out, self.offset); @@ -649,7 +649,7 @@ impl ProtocolEncode for DirectBodyRequest { /// Direct proposal-body range response (`53 + data.len()` bytes). #[derive(Clone, Debug, Eq, PartialEq)] -pub struct DirectBodyResponse { +pub struct BodyChunkResponse { pub offset: u64, pub data: Vec, pub eof: bool, @@ -657,15 +657,15 @@ pub struct DirectBodyResponse { pub body_hash: [u8; 32], } -impl DirectBodyResponse { +impl BodyChunkResponse { pub fn protocol_decode(reader: &mut ProtocolReader<'_>) -> ProtocolResult { let offset = reader.read_u64()?; - let data = reader.read_bytes_u32("DirectBodyResponse.data", DIRECT_REQUEST_MAX_BYTES)?; + let data = reader.read_bytes_u32("BodyChunkResponse.data", BODY_CHUNK_MAX_BYTES)?; let eof = reader.read_bool()?; let body_len = reader.read_u64()?; if body_len > MAX_PROPOSAL_BODY_BYTES as u64 { return Err(ProtocolError::LengthLimit { - what: "DirectBodyResponse.body_len", + what: "BodyChunkResponse.body_len", got: body_len, max: MAX_PROPOSAL_BODY_BYTES as u64, }); @@ -685,16 +685,198 @@ impl DirectBodyResponse { } } -impl ProtocolEncode for DirectBodyResponse { +impl ProtocolEncode for BodyChunkResponse { fn protocol_encode(&self, out: &mut Vec) { write_u64(out, self.offset); - write_bytes_u32(out, &self.data).expect("DirectBodyResponse data length fits u32"); + write_bytes_u32(out, &self.data).expect("BodyChunkResponse data length fits u32"); write_bool(out, self.eof); write_u64(out, self.body_len); write_fixed(out, &self.body_hash); } } +pub const PROPOSAL_LOOKUP_MAX_PAIRS: usize = 256; +pub const PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR: usize = 2; +/// chain_id(8) + realm_id(4) + pair_count(4) + pair_count * (old_root(32) + new_root(32)). +pub const PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES: usize = 16 + PROPOSAL_LOOKUP_MAX_PAIRS * 64; +pub const PROPOSAL_LOOKUP_WINDOW_PAIRS: usize = + (MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES - 5) / (64 + 1 + PROPOSAL_WIRE_BYTES); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct RealmTransition { + pub old_root: [u8; 32], + pub new_root: [u8; 32], +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProposalLookupStatus { + Candidates = 0, + Empty = 1, + Truncated = 2, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProposalLookupRequest { + pub chain_id: u64, + pub realm_id: u32, + pub pairs: Vec, +} + +impl ProposalLookupRequest { + pub fn protocol_decode(reader: &mut ProtocolReader<'_>) -> ProtocolResult { + let chain_id = reader.read_u64()?; + let realm_id = reader.read_u32()?; + let count = reader.read_u32()? as usize; + if count == 0 || count > PROPOSAL_LOOKUP_MAX_PAIRS { + return Err(ProtocolError::Message("invalid ProposalLookup pair count")); + } + if reader.remaining() < count * 64 { + return Err(ProtocolError::unexpected_eof("ProposalLookup pairs")); + } + let mut pairs = Vec::with_capacity(count); + for _ in 0..count { + pairs.push(RealmTransition { + old_root: reader.read_bytes_32()?, + new_root: reader.read_bytes_32()?, + }); + } + Ok(Self { chain_id, realm_id, pairs }) + } + + pub fn decode_exact(bytes: &[u8]) -> ProtocolResult { + decode_exact(bytes, Self::protocol_decode) + } +} + +impl ProtocolEncode for ProposalLookupRequest { + fn protocol_encode(&self, out: &mut Vec) { + write_u64(out, self.chain_id); + write_u32(out, self.realm_id); + write_u32(out, self.pairs.len() as u32); + for pair in &self.pairs { + write_fixed(out, &pair.old_root); + write_fixed(out, &pair.new_root); + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProposalLookupEntry { + pub transition: RealmTransition, + pub candidates: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProposalLookupResponse { + pub status: ProposalLookupStatus, + pub entries: Vec, +} + +impl ProposalLookupResponse { + pub fn candidates(entries: Vec) -> Self { + let status = if entries.iter().any(|entry| !entry.candidates.is_empty()) { + ProposalLookupStatus::Candidates + } else { + ProposalLookupStatus::Empty + }; + Self { status, entries } + } + + pub fn empty() -> Self { + Self { status: ProposalLookupStatus::Empty, entries: Vec::new() } + } + + pub fn truncated(entries: Vec) -> Self { + Self { status: ProposalLookupStatus::Truncated, entries } + } + + pub fn protocol_decode(reader: &mut ProtocolReader<'_>) -> ProtocolResult { + let status = match reader.read_u8()? { + 0 => ProposalLookupStatus::Candidates, + 1 => ProposalLookupStatus::Empty, + 2 => ProposalLookupStatus::Truncated, + tag => return Err(ProtocolError::UnknownTag { ty: "ProposalLookupStatus", tag }), + }; + let count = reader.read_u32()? as usize; + if count > PROPOSAL_LOOKUP_MAX_PAIRS { + return Err(ProtocolError::Message("invalid ProposalLookup entry count")); + } + if reader.remaining() < count * 65 { + return Err(ProtocolError::unexpected_eof("ProposalLookup entries")); + } + let mut entries = Vec::with_capacity(count); + let mut wire_bytes = 5; + let mut has_candidates = false; + for _ in 0..count { + let transition = RealmTransition { + old_root: reader.read_bytes_32()?, + new_root: reader.read_bytes_32()?, + }; + let candidate_count = reader.read_u8()? as usize; + if candidate_count > PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR { + return Err(ProtocolError::Message("invalid ProposalLookup candidate count")); + } + wire_bytes += 65 + candidate_count * PROPOSAL_WIRE_BYTES; + if wire_bytes > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { + return Err(ProtocolError::Message("ProposalLookup response exceeds maximum")); + } + let mut candidates = Vec::with_capacity(candidate_count); + for _ in 0..candidate_count { + candidates.push(Proposal::protocol_decode(reader)?); + } + has_candidates |= !candidates.is_empty(); + entries.push(ProposalLookupEntry { transition, candidates }); + } + if (status == ProposalLookupStatus::Candidates && !has_candidates) + || (status == ProposalLookupStatus::Empty && has_candidates) + { + return Err(ProtocolError::Message("non-canonical ProposalLookup status")); + } + Ok(Self { status, entries }) + } + + pub fn decode_exact(bytes: &[u8]) -> ProtocolResult { + if bytes.len() > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { + return Err(ProtocolError::LengthLimit { + what: "ProposalLookupResponse", + got: bytes.len() as u64, + max: MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES as u64, + }); + } + decode_exact(bytes, Self::protocol_decode) + } +} + +impl ProtocolEncode for ProposalLookupResponse { + fn protocol_encode(&self, out: &mut Vec) { + let mut wire_bytes = 5; + let mut count = 0; + for entry in self.entries.iter().take(PROPOSAL_LOOKUP_MAX_PAIRS) { + let entry_bytes = 65 + entry.candidates.len() * PROPOSAL_WIRE_BYTES; + if wire_bytes + entry_bytes > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { + break; + } + wire_bytes += entry_bytes; + count += 1; + } + let status = if count < self.entries.len() { + ProposalLookupStatus::Truncated + } else { + self.status + }; + write_u8(out, status as u8); + write_u32(out, count as u32); + for entry in &self.entries[..count] { + write_fixed(out, &entry.transition.old_root); + write_fixed(out, &entry.transition.new_root); + write_u8(out, entry.candidates.len() as u8); + for candidate in &entry.candidates { + candidate.protocol_encode(out); + } + } + } +} + /// EndCap forward stream header (exactly 60 bytes): /// `chain_id(8) + realm_id(4) + checkpoint_id(8) + end_cap_id(32) /// + end_cap_input_len(4) + proof_len(4)`. @@ -890,10 +1072,11 @@ mod tests { use super::super::domains::{DOMAIN_END_CAP_FORWARD, DOMAIN_PROPOSAL}; use super::super::codec::{sha256, validate_hash32_canonical, write_fixed, write_u16, write_u32, write_u64, GOLDILOCKS_MODULUS}; use super::super::limits::{ - CERTIFICATE_WIRE_BYTES, DIRECT_BODY_REQUEST_WIRE_BYTES, DIRECT_REQUEST_MAX_BYTES, - END_CAP_FORWARD_HEADER_WIRE_BYTES, MAX_BACKUP_BYTES, MAX_FINALIZER_OUTPUT_BYTES, - MAX_FINALIZER_PROOF_BYTES, MAX_PROPOSAL_BODY_BYTES, MAX_PROPOSAL_CHUNK_BYTES, - MAX_PROPOSAL_PARTS, PROPOSAL_WIRE_BYTES, VOTE_WIRE_BYTES, + CERTIFICATE_WIRE_BYTES, BODY_CHUNK_REQUEST_WIRE_BYTES, BODY_CHUNK_MAX_BYTES, + END_CAP_FORWARD_HEADER_WIRE_BYTES, END_CAP_FORWARD_RESPONSE_WIRE_BYTES, MAX_BACKUP_BYTES, + MAX_FINALIZER_OUTPUT_BYTES, MAX_FINALIZER_PROOF_BYTES, MAX_PROPOSAL_BODY_BYTES, + MAX_PROPOSAL_CHUNK_BYTES, MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, MAX_PROPOSAL_PARTS, + PROPOSAL_WIRE_BYTES, VOTE_WIRE_BYTES, }; use psy_core::constants::chain_id::PSY_CHAIN_ID_LOCAL_DEVNET as NETWORK_MAGIC; @@ -1230,31 +1413,31 @@ mod tests { } #[test] - fn direct_body_roundtrip() { + fn body_chunk_roundtrip() { let p = sample_proposal(); - let req = DirectBodyRequest { + let req = BodyChunkRequest { proposal_id: p.proposal_id, offset: 0, - max_bytes: DIRECT_REQUEST_MAX_BYTES, + max_bytes: BODY_CHUNK_MAX_BYTES, }; let enc = req.protocol_encode_to_vec(); - assert_eq!(enc.len(), DIRECT_BODY_REQUEST_WIRE_BYTES); + assert_eq!(enc.len(), BODY_CHUNK_REQUEST_WIRE_BYTES); assert_eq!(enc.len(), 44); - assert_eq!(DirectBodyRequest::decode_exact(&enc).unwrap(), req); - let zero_max = DirectBodyRequest { + assert_eq!(BodyChunkRequest::decode_exact(&enc).unwrap(), req); + let zero_max = BodyChunkRequest { proposal_id: p.proposal_id, offset: 0, max_bytes: 0, }; - assert!(DirectBodyRequest::decode_exact(&zero_max.protocol_encode_to_vec()).is_err()); - let over_max = DirectBodyRequest { + assert!(BodyChunkRequest::decode_exact(&zero_max.protocol_encode_to_vec()).is_err()); + let over_max = BodyChunkRequest { proposal_id: p.proposal_id, offset: 0, - max_bytes: DIRECT_REQUEST_MAX_BYTES + 1, + max_bytes: BODY_CHUNK_MAX_BYTES + 1, }; - assert!(DirectBodyRequest::decode_exact(&over_max.protocol_encode_to_vec()).is_err()); + assert!(BodyChunkRequest::decode_exact(&over_max.protocol_encode_to_vec()).is_err()); - let resp = DirectBodyResponse { + let resp = BodyChunkResponse { offset: 0, data: vec![9u8; 32], eof: true, @@ -1263,7 +1446,7 @@ mod tests { }; let renc = resp.protocol_encode_to_vec(); assert_eq!(renc.len(), 53 + 32); - assert_eq!(DirectBodyResponse::decode_exact(&renc).unwrap(), resp); + assert_eq!(BodyChunkResponse::decode_exact(&renc).unwrap(), resp); } @@ -1288,4 +1471,125 @@ mod tests { assert_ne!(id, compute_end_cap_id(NETWORK_MAGIC, 2, 3, &proof_hash, &input_hash)); assert_ne!(id, compute_end_cap_id(NETWORK_MAGIC ^ (1u64 << 32), 2, 3, &input_hash, &proof_hash)); } + + #[test] + fn history_wire_regression() { + let proposal = sample_proposal(); + assert_eq!(proposal.protocol_encode_to_vec().len(), 214); + let vote = Vote::new(proposal.proposal_id, 1, BlsSecretKey::key_gen(&[1u8; 32]).unwrap().sign_vote(&vote_message( + NETWORK_MAGIC, 2, &proposal.validator_tree_root, &proposal.proposal_id, + ))); + assert_eq!(vote.protocol_encode_to_vec().len(), 130); + let cert = Certificate { + chain_id: NETWORK_MAGIC, + realm_id: 2, + validator_tree_root: proposal.validator_tree_root, + proposal_id: proposal.proposal_id, + signer_bitmap: [0u8; 32], + aggregated_signature: BlsSecretKey::key_gen(&[1u8; 32]).unwrap().sign_vote(&[0u8; 8]), + }; + assert_eq!(cert.protocol_encode_to_vec().len(), 204); + assert_eq!(END_CAP_FORWARD_HEADER_WIRE_BYTES, 60); + assert_eq!(END_CAP_FORWARD_RESPONSE_WIRE_BYTES, 18); + let body = encode_proposal_body( + &vec![0u8; MAX_FINALIZER_OUTPUT_BYTES], + &[0u8; 1], + &[0u8; 1], + &[0x11u8; 32], + ).unwrap(); + assert_eq!(&body[body.len() - 32..], &[0x11u8; 32]); + let proposal_bytes = proposal.protocol_encode_to_vec(); + assert!(Proposal::decode_exact(&proposal_bytes[..213]).is_err()); + assert!(Vote::decode_exact(&vote.protocol_encode_to_vec()[..129]).is_err()); + assert!(Certificate::decode_exact(&cert.protocol_encode_to_vec()[..203]).is_err()); + } + +} + +#[cfg(test)] +mod proposal_lookup_tests { + use super::*; + + fn pair(index: u8) -> RealmTransition { + RealmTransition { old_root: [index; 32], new_root: [index.wrapping_add(1); 32] } + } + + fn entries(count: usize, candidates: usize) -> Vec { + let proposal = proposal_from_parts(1, 2, 3, 0, [1; 32], [2; 32], [3; 32], [4; 32], [5; 32]); + (0..count).map(|index| ProposalLookupEntry { + transition: pair(index as u8), + candidates: vec![proposal.clone(); candidates], + }).collect() + } + + #[test] + fn request_window_bounds_and_canonical_length() { + let mut request = ProposalLookupRequest { + chain_id: 1, realm_id: 2, + pairs: (0..PROPOSAL_LOOKUP_MAX_PAIRS).map(|index| pair(index as u8)).collect(), + }; + let bytes = request.protocol_encode_to_vec(); + assert_eq!(bytes.len(), 16 + 256 * 64); + assert_eq!(bytes.len(), PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES); + assert_eq!(ProposalLookupRequest::decode_exact(&bytes).unwrap(), request); + assert!(ProposalLookupRequest::decode_exact(&bytes[..bytes.len() - 1]).is_err()); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(ProposalLookupRequest::decode_exact(&trailing).is_err()); + for count in [0u32, 257, u32::MAX] { + let mut invalid = bytes.clone(); + invalid[12..16].copy_from_slice(&count.to_le_bytes()); + assert!(ProposalLookupRequest::decode_exact(&invalid).is_err()); + } + request.pairs.truncate(1); + assert_eq!(request.protocol_encode_to_vec().len(), 80); + assert_eq!(ProposalLookupRequest::decode_exact(&request.protocol_encode_to_vec()).unwrap(), request); + } + + #[test] + fn response_budget_preserves_complete_prefix() { + assert_eq!(PROPOSAL_LOOKUP_WINDOW_PAIRS, 58); + let response = ProposalLookupResponse::candidates(entries(PROPOSAL_LOOKUP_WINDOW_PAIRS, 1)); + let bytes = response.protocol_encode_to_vec(); + assert_eq!(bytes.len(), 5 + 58 * (65 + PROPOSAL_WIRE_BYTES)); + assert!(bytes.len() <= MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES); + assert_eq!(ProposalLookupResponse::decode_exact(&bytes).unwrap(), response); + for candidate_count in [0, 1, 2] { + let response = ProposalLookupResponse::candidates(entries(256, candidate_count)); + let bytes = response.protocol_encode_to_vec(); + assert!(bytes.len() <= MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES); + let decoded = ProposalLookupResponse::decode_exact(&bytes).unwrap(); + assert_eq!(decoded.status, ProposalLookupStatus::Truncated); + let count = (MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES - 5) / (65 + candidate_count * PROPOSAL_WIRE_BYTES); + assert_eq!(decoded.entries, response.entries[..count]); + } + } + + #[test] + fn response_rejects_invalid_counts_status_and_suffixes() { + let response = ProposalLookupResponse::candidates(entries(1, 2)); + let bytes = response.protocol_encode_to_vec(); + assert_eq!(ProposalLookupResponse::decode_exact(&bytes).unwrap(), response); + for status in [1, 3, 255] { + let mut invalid = bytes.clone(); + invalid[0] = status; + assert!(ProposalLookupResponse::decode_exact(&invalid).is_err()); + } + let mut invalid = bytes.clone(); + invalid[69] = 3; + assert!(ProposalLookupResponse::decode_exact(&invalid).is_err()); + invalid = bytes.clone(); + invalid[1..5].copy_from_slice(&257u32.to_le_bytes()); + assert!(ProposalLookupResponse::decode_exact(&invalid).is_err()); + assert!(ProposalLookupResponse::decode_exact(&bytes[..bytes.len() - 1]).is_err()); + invalid = bytes; + invalid.push(0); + assert!(ProposalLookupResponse::decode_exact(&invalid).is_err()); + for response in [ProposalLookupResponse::empty(), ProposalLookupResponse::truncated(Vec::new()), ProposalLookupResponse::candidates(entries(1, 0))] { + assert_eq!(ProposalLookupResponse::decode_exact(&response.protocol_encode_to_vec()).unwrap(), response); + } + let mut invalid = ProposalLookupResponse::empty().protocol_encode_to_vec(); + invalid[0] = ProposalLookupStatus::Candidates as u8; + assert!(ProposalLookupResponse::decode_exact(&invalid).is_err()); + } } diff --git a/psy_data/src/p2p/mod.rs b/psy_data/src/p2p/mod.rs index d366666c..5197d841 100644 --- a/psy_data/src/p2p/mod.rs +++ b/psy_data/src/p2p/mod.rs @@ -32,9 +32,11 @@ pub use error::{ProtocolError, ProtocolResult}; pub use limits::*; pub use messages::{ bitmap_get, bitmap_set, compute_end_cap_id, compute_proposal_id, encode_proposal_body, - proposal_from_parts, vote_message, Certificate, DirectBodyRequest, DirectBodyResponse, - EndCapForwardHeader, EndCapForwardResponse, EndCapRejectReason, Proposal, ProposalPart, - RealmFinalizeOutputBytes, RealmFinalizeSubmitCode, Vote, + proposal_from_parts, vote_message, Certificate, BodyChunkRequest, BodyChunkResponse, + EndCapForwardHeader, EndCapForwardResponse, EndCapRejectReason, Proposal, ProposalLookupEntry, + ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, ProposalPart, + RealmFinalizeOutputBytes, RealmFinalizeSubmitCode, Vote, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, + PROPOSAL_LOOKUP_MAX_PAIRS, PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES, PROPOSAL_LOOKUP_WINDOW_PAIRS, }; pub use node_id::NodeId; pub use validator_leaf::ValidatorLeaf; diff --git a/psy_data/src/queue_items/realm_user_update.rs b/psy_data/src/queue_items/realm_user_update.rs index c4c6274b..2724b097 100644 --- a/psy_data/src/queue_items/realm_user_update.rs +++ b/psy_data/src/queue_items/realm_user_update.rs @@ -11,7 +11,7 @@ use crate::{guta::stats::GUTAStats, proof_input::guta::end_cap_input::PsyUserEve #[ts(export, concrete(F = parth_core::PF, Hash = parth_core::PHash))] pub struct PsyRealmUserUpdateQueueItem { pub job_id: QProvingJobDataID, - pub expected_fake_checkpoint_id: u64, + pub submission_nonce: u64, pub old_user_leaf_hash: Hash, pub new_user_leaf_hash: Hash, pub new_user_leaf: PQEDUserLeaf, @@ -22,7 +22,7 @@ pub struct PsyRealmUserUpdateQueueItem { impl PsyRealmUserUpdateQueueItem { pub fn new( job_id: QProvingJobDataID, - expected_fake_checkpoint_id: u64, + submission_nonce: u64, old_user_leaf_hash: Hash, new_user_leaf_hash: Hash, new_user_leaf: PQEDUserLeaf, @@ -31,7 +31,7 @@ impl PsyRealmUserUpdateQueueItem { ) -> Self { Self { job_id, - expected_fake_checkpoint_id, + submission_nonce, old_user_leaf_hash, new_user_leaf_hash, new_user_leaf, @@ -48,7 +48,7 @@ impl QPGenRandom for PsyRealmUserUpdateQueueI { PsyRealmUserUpdateQueueItem { job_id: QProvingJobDataID::qp_rand_gen(), - expected_fake_checkpoint_id: u64::qp_rand_gen(), + submission_nonce: u64::qp_rand_gen(), old_user_leaf_hash: Hash::qp_rand_gen(), new_user_leaf_hash: Hash::qp_rand_gen(), new_user_leaf: PQEDUserLeaf::qp_rand_gen(), @@ -76,7 +76,7 @@ impl FallbackPsySerializeCanonical for PsyRealmUs fn fallback_pio_write_to_io(&self, writer: &mut W) -> anyhow::Result<()> { writer.psy_write_bytes_fixed(&self.job_id.to_fixed_bytes())?; - writer.psy_write_u64(self.expected_fake_checkpoint_id)?; + writer.psy_write_u64(self.submission_nonce)?; writer.psy_write_bytes_fixed(&self.old_user_leaf_hash.into_owned_32bytes())?; writer.psy_write_bytes_fixed(&self.new_user_leaf_hash.into_owned_32bytes())?; self.new_user_leaf.pio_write_to_io(writer)?; @@ -91,7 +91,7 @@ impl FallbackPsySerializeCanonical for PsyRealmUs fn fallback_pio_read_from_io(reader: &mut R) -> anyhow::Result { let job_id = QProvingJobDataID::try_from_byte_vec(&reader.psy_read_bytes_fixed::()?)?; - let expected_fake_checkpoint_id = reader.psy_read_u64()?; + let submission_nonce = reader.psy_read_u64()?; let old_user_leaf_hash = Hash::from_owned_32bytes(reader.psy_read_bytes_fixed()?); let new_user_leaf_hash = Hash::from_owned_32bytes(reader.psy_read_bytes_fixed()?); let new_user_leaf = PQEDUserLeaf::::pio_read_from_io(reader)?; @@ -103,7 +103,7 @@ impl FallbackPsySerializeCanonical for PsyRealmUs } Ok(Self { job_id, - expected_fake_checkpoint_id, + submission_nonce, old_user_leaf_hash, new_user_leaf_hash, new_user_leaf, @@ -130,7 +130,7 @@ pser::impl_psy_ser_basic_tests_fallback!( impl PCoreQueueItemBase for PsyRealmUserUpdateQueueItem { fn is_queue_item(data: &[u8]) -> bool { // Variable-length payload: - // fixed prefix = job_id + expected_fake_checkpoint_id + 2*hash + user_leaf + stats + events_len(u32) + // fixed prefix = job_id + submission_nonce + 2*hash + user_leaf + stats + events_len(u32) let min_size = QJOB_ID_SERIALIZED_SIZE + 8 + 32 diff --git a/psy_node_common/src/backup/global_user_tree/db_loader_sub_root.rs b/psy_node_common/src/backup/global_user_tree/db_loader_sub_root.rs index dd0db8db..eaab6277 100644 --- a/psy_node_common/src/backup/global_user_tree/db_loader_sub_root.rs +++ b/psy_node_common/src/backup/global_user_tree/db_loader_sub_root.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use cf_utils::timer::TraceTimer; use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; +use parth_common::tree_sync::traits::rehash_sparse_paths; use parth_core::{crypto::hash::traits::MerkleZeroHasher, data::hash::merkle_node_key::SimpleMerkleNodeKey}; use psy_node_core::psy_core_db::traits::full::PsyNodeGlobalUserTreeDatabaseReader; @@ -32,68 +33,53 @@ pub async fn fetch_global_user_tree_from_db_with_sub_root< fetch_batch_size ); let mut node_hash_map = HashMap::::new(); - let total = max_user_id_exclusive - min_user_id_inclusive; + if sub_root.level > tree_height { + anyhow::bail!("sub root {:?} level exceeds tree_height {}", sub_root, tree_height); + } + if min_user_id_inclusive > max_user_id_exclusive { + anyhow::bail!("invalid user ID range [{}, {})", min_user_id_inclusive, max_user_id_exclusive); + } + let shift = tree_height - sub_root.level; + let sub_tree_width = 1u64 + .checked_shl(shift as u32) + .ok_or_else(|| anyhow::anyhow!("sub-root level shift overflow for {:?}", sub_root))?; + let leaf_min_index = sub_root + .index + .checked_mul(sub_tree_width) + .ok_or_else(|| anyhow::anyhow!("sub-root index shift overflow for {:?}", sub_root))?; + let leaf_max_exclusive = leaf_min_index + .checked_add(sub_tree_width) + .ok_or_else(|| anyhow::anyhow!("sub-root leaf range overflow for {:?}", sub_root))?; + if leaf_min_index > min_user_id_inclusive || leaf_max_exclusive < max_user_id_exclusive { + anyhow::bail!("Sub root {:?} does not cover the requested user ID range [{}, {})", sub_root, min_user_id_inclusive, max_user_id_exclusive); + } + let sub_tree_leaf_level = shift; // DB returns zero leaves as the leaf-level zero hash, not the sub-tree-root level zero. let leaf_zero_hash = Hasher::get_zero_hash(0); - let full_batches = total / fetch_batch_size as u64; - let remainder = total % fetch_batch_size as u64; - let batch_capacity = if full_batches > 0 { fetch_batch_size } else { remainder as usize }; - let mut keys = vec![ - SimpleMerkleNodeKey { - level: tree_height, - index: 0, - }; - batch_capacity - ]; timer.start(); - let leaf_min_index = sub_root.index << (tree_height - sub_root.level); - if leaf_min_index > min_user_id_inclusive || leaf_min_index + (1u64 << (tree_height - sub_root.level)) < max_user_id_exclusive { - anyhow::bail!("Sub root {:?} does not cover the requested user ID range [{}, {})", sub_root, min_user_id_inclusive, max_user_id_exclusive); - } - let sub_tree_leaf_level = tree_height - sub_root.level; - for batch_index in 0..full_batches { - let start_user_id = min_user_id_inclusive + batch_index * fetch_batch_size as u64; - // Reset level on every batch: the post-fetch loop below mutates keys[i].level - // for hashmap insertion, and we must re-issue DB lookups at the original leaf level. - for i in 0..fetch_batch_size { - keys[i] = SimpleMerkleNodeKey { - level: tree_height, - index: start_user_id + i as u64, - }; - } - let batch_results = user_db_reader.global_user_tree_get_nodes(checkpoint_id, &keys).await?; - for (i, hash) in batch_results.iter().enumerate() { - if hash == &leaf_zero_hash { - continue; - } - let local_key = SimpleMerkleNodeKey { - level: sub_tree_leaf_level, - index: keys[i].index - leaf_min_index, - }; - node_hash_map.insert(local_key, *hash); - } - } - if remainder > 0 { - let start_user_id = min_user_id_inclusive + full_batches * fetch_batch_size as u64; - for i in 0..remainder as usize { - keys[i] = SimpleMerkleNodeKey { - level: tree_height, - index: start_user_id + i as u64, - }; + let dumped = user_db_reader + .global_user_tree_dump_leaves_range(checkpoint_id, min_user_id_inclusive, max_user_id_exclusive) + .await?; + for (index, hash) in dumped { + if index < min_user_id_inclusive || index >= max_user_id_exclusive { + anyhow::bail!( + "range dump returned index {} outside requested [{}, {})", + index, + min_user_id_inclusive, + max_user_id_exclusive + ); } - let batch_results = user_db_reader - .global_user_tree_get_nodes(checkpoint_id, &keys[0..remainder as usize]) - .await?; - for (i, hash) in batch_results.iter().enumerate() { - if hash == &leaf_zero_hash { - continue; - } - let local_key = SimpleMerkleNodeKey { - level: sub_tree_leaf_level, - index: keys[i].index - leaf_min_index, - }; - node_hash_map.insert(local_key, *hash); + if hash == leaf_zero_hash { + continue; } + let local_index = index + .checked_sub(leaf_min_index) + .ok_or_else(|| anyhow::anyhow!("range dump index {} is below sub-root leaf min {}", index, leaf_min_index))?; + let local_key = SimpleMerkleNodeKey { + level: sub_tree_leaf_level, + index: local_index, + }; + node_hash_map.insert(local_key, hash); } timer.lap_batch( "fetched global user tree nodes from DB", @@ -112,11 +98,12 @@ pub async fn fetch_global_user_tree_from_db_with_sub_root< max_user_id_exclusive ); - let mut tree = SimpleMemoryMerkleRecorderStore::from_hash_map(tree_height-sub_root.level, node_hash_map); + let sparse_keys: Vec = node_hash_map.keys().copied().collect(); + let mut tree = SimpleMemoryMerkleRecorderStore::from_hash_map(shift, node_hash_map); timer.start(); - tree.rehash_range(tree_height-sub_root.level, min_user_id_inclusive-leaf_min_index, max_user_id_exclusive-leaf_min_index); - timer.lap_batch("rehashed global user tree nodes", "node", (max_user_id_exclusive - min_user_id_inclusive) as usize); + rehash_sparse_paths(&mut tree, &sparse_keys, 0); + timer.lap_batch("rehashed sparse leaf paths", "path", sparse_keys.len()); tree.commit_changes(); timer.lap("committed changes to memory global user tree"); @@ -218,10 +205,11 @@ mod tests { merkle_node_key::SimpleMerkleNodeKey, }, }; + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; use parth_crypto::hash::sha256::CoreSha256Hasher; use psy_node_core::psy_core_db::traits::full::PsyNodeGlobalUserTreeDatabaseReader; - use super::fetch_global_user_tree_from_db_with_sub_root; + use super::{fetch_global_user_tree_from_db_with_sub_root, load_global_user_tree_from_db_with_sub_root}; /// Minimal in-memory reader: stores leaves and computes parents on demand using /// the same hasher the loader uses. Returns the leaf-level zero hash for absent @@ -229,17 +217,27 @@ mod tests { struct MockUserTreeReader { height: u8, leaves: Mutex>, + node_overrides: Mutex>, } impl MockUserTreeReader { fn new(height: u8) -> Self { - Self { height, leaves: Mutex::new(HashMap::new()) } + Self { + height, + leaves: Mutex::new(HashMap::new()), + node_overrides: Mutex::new(HashMap::new()), + } } fn set_leaf(&self, index: u64, value: Hash256) { self.leaves.lock().unwrap().insert(index, value); } + fn set_node_override(&self, key: SimpleMerkleNodeKey, value: Hash256) { + self.node_overrides.lock().unwrap().insert(key, value); + } fn node(&self, key: SimpleMerkleNodeKey) -> Hash256 { - // Recursively compute the hash at (level, index) by hashing children. + if let Some(value) = self.node_overrides.lock().unwrap().get(&key).copied() { + return value; + } if key.level == self.height { return self .leaves @@ -249,7 +247,6 @@ mod tests { .copied() .unwrap_or_else(|| >::get_zero_hash(0)); } - // Internal node: hash(children). let left = self.node(SimpleMerkleNodeKey::new(key.level + 1, key.index << 1)); let right = self.node(SimpleMerkleNodeKey::new(key.level + 1, (key.index << 1) | 1)); >::two_to_one(&left, &right) @@ -285,6 +282,23 @@ mod tests { async fn global_user_tree_dump_all_leaves(&self, _cp: u64) -> anyhow::Result> { unimplemented!() } + async fn global_user_tree_dump_leaves_range( + &self, + _cp: u64, + min_user_id_inclusive: u64, + max_user_id_exclusive: u64, + ) -> anyhow::Result> { + if min_user_id_inclusive >= max_user_id_exclusive { + return Ok(HashMap::new()); + } + let zero = >::get_zero_hash(0); + let leaves = self.leaves.lock().unwrap(); + Ok(leaves + .iter() + .filter(|(index, hash)| **index >= min_user_id_inclusive && **index < max_user_id_exclusive && **hash != zero) + .map(|(index, hash)| (*index, *hash)) + .collect()) + } async fn global_user_tree_get_node_and_checkpoint_id_max_checkpoint( &self, _max_cp: u64, @@ -375,4 +389,180 @@ mod tests { assert_eq!(tree.get_root(), expected_root); } + + #[tokio::test(flavor = "current_thread")] + async fn empty_range_returns_empty_tree() { + let tree_height: u8 = 8; + let sub_root_key = SimpleMerkleNodeKey { level: 4, index: 1 }; + let leaf_min = 1u64 << 4; + let reader = MockUserTreeReader::new(tree_height); + let tree = fetch_global_user_tree_from_db_with_sub_root::( + &reader, + tree_height, + sub_root_key, + 1, + leaf_min, + leaf_min, + 8, + ) + .await + .expect("empty range should succeed"); + assert_eq!( + tree.get_root(), + >::get_zero_hash(4) + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn current_overwrite_then_zero_filters_leaf_zero_hash() { + let tree_height: u8 = 8; + let sub_root_level: u8 = 4; + let sub_root_index: u64 = 2; + let leaf_min = sub_root_index << (tree_height - sub_root_level); + let reader = MockUserTreeReader::new(tree_height); + let idx = leaf_min + 3; + let first = Hash256::from_u64_le_values(11, 0, 0, 0); + let second = Hash256::from_u64_le_values(22, 0, 0, 0); + reader.set_leaf(idx, first); + reader.set_leaf(idx, second); + let sub_root_key = SimpleMerkleNodeKey { level: sub_root_level, index: sub_root_index }; + let expected = reader.node(sub_root_key); + let tree = fetch_global_user_tree_from_db_with_sub_root::( + &reader, + tree_height, + sub_root_key, + 9, + leaf_min, + leaf_min + 16, + 4, + ) + .await + .expect("overwrite fetch"); + assert_eq!(tree.get_root(), expected); + + reader.set_leaf(idx, >::get_zero_hash(0)); + let expected_zeroed = reader.node(sub_root_key); + let tree_zeroed = fetch_global_user_tree_from_db_with_sub_root::( + &reader, + tree_height, + sub_root_key, + 9, + leaf_min, + leaf_min + 16, + 4, + ) + .await + .expect("zeroed fetch"); + assert_eq!(tree_zeroed.get_root(), expected_zeroed); + } + + #[tokio::test(flavor = "current_thread")] + async fn nonzero_realm_offset_loads_local_indexes() { + let tree_height: u8 = 8; + let sub_root_level: u8 = 4; + let sub_root_index: u64 = 3; + let leaf_min = sub_root_index << (tree_height - sub_root_level); + let reader = MockUserTreeReader::new(tree_height); + reader.set_leaf(leaf_min + 1, Hash256::from_u64_le_values(3, 1, 4, 1)); + reader.set_leaf(leaf_min + 15, Hash256::from_u64_le_values(5, 9, 2, 6)); + let sub_root_key = SimpleMerkleNodeKey { level: sub_root_level, index: sub_root_index }; + let expected = reader.node(sub_root_key); + let tree = fetch_global_user_tree_from_db_with_sub_root::( + &reader, + tree_height, + sub_root_key, + 4, + leaf_min, + leaf_min + 16, + 7, + ) + .await + .expect("offset realm"); + assert_eq!(tree.get_root(), expected); + } + + /// Corrupt leaf inside the dumped range, with only the sub-root node + /// overridden to the honest value. No parent nodes are stored. Loader's + /// root-equality check must fail. + #[tokio::test(flavor = "current_thread")] + async fn corrupt_dumped_leaf_mismatches_honest_sub_root() { + let tree_height: u8 = 8; + let sub_root_level: u8 = 4; + let sub_root_index: u64 = 1; + let leaf_min = sub_root_index << (tree_height - sub_root_level); + let honest = MockUserTreeReader::new(tree_height); + honest.set_leaf(leaf_min + 2, Hash256::from_u64_le_values(8, 8, 8, 8)); + let sub_root_key = SimpleMerkleNodeKey { level: sub_root_level, index: sub_root_index }; + let honest_root = honest.node(sub_root_key); + + let corrupt = MockUserTreeReader::new(tree_height); + corrupt.set_leaf(leaf_min + 2, Hash256::from_u64_le_values(9, 9, 9, 9)); + corrupt.set_node_override(sub_root_key, honest_root); + + let err = load_global_user_tree_from_db_with_sub_root::( + &corrupt, + tree_height, + sub_root_key, + 1, + 8, + ) + .await + .expect_err("corrupt leaf must fail root equality"); + let msg = format!("{err:#}"); + assert!(msg.contains("does not match DB sub_root"), "{msg}"); + } + + /// Test-only D0 dense `rehash_range` vs production sparse loader on the same leaves. + #[tokio::test(flavor = "current_thread")] + async fn old_point_batches_match_range_same_leaf_set() { + let tree_height: u8 = 8; + let sub_root_level: u8 = 4; + let sub_root_index: u64 = 1; + let leaf_min = sub_root_index << (tree_height - sub_root_level); + let reader = MockUserTreeReader::new(tree_height); + for i in 0..16u64 { + if i % 3 == 0 { + continue; + } + reader.set_leaf(leaf_min + i, Hash256::from_u64_le_values(i + 1, 2, 3, 4)); + } + let sub_root_key = SimpleMerkleNodeKey { level: sub_root_level, index: sub_root_index }; + let expected = reader.node(sub_root_key); + + let range_tree = fetch_global_user_tree_from_db_with_sub_root::( + &reader, + tree_height, + sub_root_key, + 1, + leaf_min, + leaf_min + 16, + 5, + ) + .await + .expect("range path"); + + let mut keys = Vec::with_capacity(16); + for i in 0..16u64 { + keys.push(SimpleMerkleNodeKey { level: tree_height, index: leaf_min + i }); + } + let point = reader.global_user_tree_get_nodes(1, &keys).await.unwrap(); + let zero = >::get_zero_hash(0); + let mut point_map = HashMap::new(); + for (i, hash) in point.into_iter().enumerate() { + if hash != zero { + point_map.insert( + SimpleMerkleNodeKey { level: 4, index: i as u64 }, + hash, + ); + } + } + // D0 dense writer, test-only — production loader no longer calls rehash_range. + let mut point_tree = SimpleMemoryMerkleRecorderStore::::from_hash_map(4, point_map); + point_tree.rehash_range(4, 0, 16); + + assert_eq!(range_tree.get_root(), expected); + assert_eq!(point_tree.get_root(), expected); + assert_eq!(range_tree.get_root(), point_tree.get_root()); + + } } diff --git a/psy_node_common/src/coordinator/edge/api.rs b/psy_node_common/src/coordinator/edge/api.rs index bc32f2b7..e65813bc 100644 --- a/psy_node_common/src/coordinator/edge/api.rs +++ b/psy_node_common/src/coordinator/edge/api.rs @@ -93,9 +93,8 @@ impl< _realm_id: u64, proposal: Option>, certificate: Option>, - finalize_binding: Vec, ) -> QRpcResult { - res_guta(self.submit_guta_internal(input, proof, proposal, certificate, finalize_binding).await.map_err(GutaSubmitError::from)) + res_guta(self.submit_guta_internal(input, proof, proposal, certificate).await.map_err(GutaSubmitError::from)) } async fn get_user_ids_for_public_key(&self, public_key: N::QHash, start_user_id: u64, count: u32) -> QRpcResult> { diff --git a/psy_node_common/src/coordinator/edge/handler.rs b/psy_node_common/src/coordinator/edge/handler.rs index 85e1e8ee..834f8f5b 100644 --- a/psy_node_common/src/coordinator/edge/handler.rs +++ b/psy_node_common/src/coordinator/edge/handler.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use tokio::task; use parth_core::{ - QCoreProcCheckpointUniqueId, QProvingJobDataIDWithRewardPath, crypto::hash::{merkle_proof::MerkleProofCore, tag_tree::TagTreeMerkleProof, traits::{HashTo4Felts, MerkleZeroHasher, QFieldHashable}}, data::{hash::merkle_node_key::SimpleMerkleNodeKey, queue::queue_key::QPBaseQueueType}, felt::{FromPrimitiveValuesFelt, ToU64Value}, node::realm_identifier::QRealmIdentifier, protocol::core_types::{Q256BitHash, QNetworkTypesConfig, QZKProofVerifier} + QCoreProcCheckpointUniqueId, QProvingJobDataIDWithRewardPath, crypto::hash::{merkle_proof::MerkleProofCore, tag_tree::TagTreeMerkleProof, traits::{HashTo4Felts, MerkleZeroHasher, QFieldHashable}}, data::{hash::merkle_node_key::SimpleMerkleNodeKey, queue::queue_key::QPBaseQueueType}, felt::ToU64Value, node::realm_identifier::QRealmIdentifier, protocol::core_types::{Q256BitHash, QNetworkTypesConfig, QZKProofVerifier} }; use psy_config::CHECKPOINTS_PER_EPOCH; use psy_core::job::job_id::{ProvingJobCircuitType, QProvingJobDataID}; @@ -11,10 +11,6 @@ use psy_api_core::CheckpointJobStats; use psy_data::{ guta::{ header_extended::{GlobalUserTreeAggregatorHeaderWithTagValueAndJobID, GlobalUserTreeAggregatorHeaderWithTagValueAndJobType}, - realm_finalize::{ - finalize_reward_root63, protocol_decode_finalize_output, - realm_finalize_guta_chain_domain, RealmFinalizeGUTAPublicOutput, - }, }, p2p::{sha256, Certificate, Proposal, RealmFinalizeSubmitCode}, prepared_block::realm::PsyRealmCoordinatorUpdate, @@ -38,7 +34,7 @@ use crate::{ coordinator::queue_key::{CoordinatorDeployContractQueueKey, CoordinatorRegisterUserPublicKeyQueueKey, CoordinatorSubmitRealmGUTAUpdateQueueKey, CoordinatorUpdateContractQueueKey}, p2p::guta_submit::GutaSubmitError, realm::processor::consensus::{ - certificate_includes_proposer, inclusion_lag_within_limit, + certificate_includes_proposer, require_nonzero_validator_tree_root, validate_certificate, validator_tree_root_matches_proof_base, votes_meet_wait, }, @@ -605,19 +601,12 @@ impl< } } -/// realm-finalize-bls-auth.md §5 admission shape: an external Realm GUTA -/// submission must be circuit 63 (`RealmFinalizeGUTA`) and carry a non-empty -/// finalize binding. Any other job type — with or without a binding — is -/// explicitly rejected instead of being silently admitted. -fn validate_external_guta_admission(job_type_u32: u32, finalize_binding: &[u8]) -> anyhow::Result<()> { +/// External Realm GUTA admission requires circuit 63 (`RealmFinalizeGUTA`). +fn validate_external_guta_admission(job_type_u32: u32) -> anyhow::Result<()> { anyhow::ensure!( job_type_u32 == ProvingJobCircuitType::RealmFinalizeGUTA as u32, "external Realm GUTA admission requires job type 63 (RealmFinalizeGUTA), got {job_type_u32}" ); - anyhow::ensure!( - !finalize_binding.is_empty(), - "realm finalize submission requires a finalize binding" - ); Ok(()) } @@ -650,7 +639,6 @@ impl< proof_bytes: Vec, proposal_bytes: Option>, certificate_bytes: Option>, - finalize_binding: Vec, ) -> anyhow::Result<()> where N::ZKVerifier: 'static, @@ -696,7 +684,6 @@ impl< &proof_bytes, proposal_bytes.as_deref(), certificate_bytes.as_deref(), - &finalize_binding, ) .await?; let expected_public_inputs_hash = input.qfhash::(); @@ -786,11 +773,8 @@ impl< /// Mandatory Realm-finalize admission gate. /// /// Reuses the existing tree lookup, scheduled-proposer calculation and BLS - /// FastAggregateVerify, and additionally verifies the actual finalizer - /// output carried by `finalize_binding` (442 bytes: 410-byte O + 32-byte - /// worker tag). The scheduled proposer's certificate vote over the - /// proposal identity D is the off-circuit BLS authorization; the - /// proof-bound fee user must equal that proposer's validator-leaf user ID. + /// FastAggregateVerify. The scheduled proposer's certificate vote over the + /// proposal identity D is the off-circuit BLS authorization. async fn verify_guta_certificate( &self, realm_id: u32, @@ -798,12 +782,8 @@ impl< proof_bytes: &[u8], proposal_bytes: Option<&[u8]>, certificate_bytes: Option<&[u8]>, - finalize_binding: &[u8], ) -> anyhow::Result<()> { - // realm-finalize-bls-auth.md §5: external Realm admission requires job - // type 63 with a non-empty finalize binding. Ordinary GUTA aggregation - // never crosses this edge, and no legacy non-63 path is permitted. - validate_external_guta_admission(input.job_type_u32, finalize_binding)?; + validate_external_guta_admission(input.job_type_u32)?; let proposal = Proposal::decode_exact(proposal_bytes.ok_or_else(|| { anyhow::anyhow!("rotation enabled but GUTA Proposal missing for realm {realm_id}") })?) @@ -817,33 +797,6 @@ impl< anyhow::ensure!(proposal.realm_id == realm_id, "GUTA Proposal realm mismatch"); anyhow::ensure!(proposal.finalizer_proof_hash == sha256(proof_bytes), "GUTA Proposal proof hash mismatch"); - // 1. Admission guarantees circuit 63: strictly decode the binding and - // the actual output; bind them to the submitted header/tag and the - // circuit public input. - let output = { - let binding = psy_data::guta::realm_finalize::RealmFinalizeBinding::protocol_decode(finalize_binding) - .map_err(|error| anyhow::anyhow!("invalid finalize binding: {error}"))?; - let output: RealmFinalizeGUTAPublicOutput = - protocol_decode_finalize_output(&binding.output) - .map_err(|error| anyhow::anyhow!("invalid Realm finalize output: {error}"))?; - anyhow::ensure!( - output.final_guta_header == input.header.header, - "finalize output final header does not match the submitted header" - ); - let worker_tag = N::QHash::from_owned_32bytes(binding.finalizer_worker_reward_tag); - let reward_root63 = finalize_reward_root63::(&output, &worker_tag); - anyhow::ensure!( - reward_root63 == input.header.new_tag_tree_node_value, - "finalize output does not bind the submitted reward tag (A/R63 mismatch)" - ); - anyhow::ensure!( - proposal.public_output_hash == sha256(&binding.output), - "GUTA Proposal public output hash mismatch" - ); - Some(output) - }; - - // 2. Canonical proof base, chain domain and field consistency. let canonical_base_checkpoint_id = self .db_reader .get_checkpoint_id_for_checkpoint_root_hash(input.header.header.checkpoint_tree_root) @@ -853,42 +806,12 @@ impl< proposal.base_checkpoint_id == canonical_base_checkpoint_id, "GUTA Proposal proof-base checkpoint does not match submitted checkpoint tree root" ); - if let Some(output) = &output { - anyhow::ensure!( - output.checkpoint_id == N::F::from_u64_value(canonical_base_checkpoint_id), - "finalize output checkpoint_id does not match the canonical proof base" - ); - let expected_chain_domain = - realm_finalize_guta_chain_domain::(self.chain_id); - anyhow::ensure!( - output.chain_domain == expected_chain_domain, - "finalize output chain domain mismatch" - ); - anyhow::ensure!( - output.realm_id == N::F::from_u64_value(realm_id as u64), - "finalize output realm mismatch" - ); - anyhow::ensure!( - output.validator_tree_root.into_owned_32bytes() == proposal.validator_tree_root, - "finalize output validator_tree_root mismatch" - ); - } let proof_base_roots = self .db_reader .get_checkpoint_global_state_roots(proposal.base_checkpoint_id) .await?; - if let Some(output) = &output { - anyhow::ensure!( - output.validator_tree_root.into_owned_32bytes() - == proof_base_roots.validator_tree_root.into_owned_32bytes(), - "finalize output validator tree root does not match the proof-base checkpoint" - ); - } - // 3. Load checkpoint-authenticated validators; preimage chain ID must - // equal the configured chain ID (defense in depth beyond the circuit - // validator-tree proof). - let (validator_sub_ids, keys, user_ids) = load_realm_validators_from_tree::( + let (validator_sub_ids, keys, _user_ids, _) = load_realm_validators_from_tree::( &*self.db_reader, self.chain_id, proposal.base_checkpoint_id, @@ -914,30 +837,6 @@ impl< "GUTA Proposal validator_tree_root does not match proof-base checkpoint" ); - // 4./5. Scheduled proposer for T = P + 1 and the fee-claim glue: the - // proof-bound output fee user must be the scheduled proposer's - // authenticated validator-leaf user ID. - let inclusion_checkpoint_id = self - .get_latest_checkpoint_id_internal() - .await? - .checked_add(1) - .ok_or_else(|| anyhow::anyhow!("latest checkpoint ID overflow at GUTA admission"))?; - let inclusion_lag = inclusion_lag_within_limit( - proposal.base_checkpoint_id, - inclusion_checkpoint_id, - ) - .ok_or_else(|| { - GutaSubmitError::retryable( - RealmFinalizeSubmitCode::InvalidProposal, - format!( - "GUTA Proposal proof-base checkpoint {} is not within the maximum inclusion lag {} at inclusion checkpoint {}", - proposal.base_checkpoint_id, - psy_data::p2p::MAX_INCLUSION_LAG_CHECKPOINTS, - inclusion_checkpoint_id - ), - ) - })?; - let _ = inclusion_lag; let target_checkpoint_id = proposal .base_checkpoint_id .checked_add(1) @@ -970,25 +869,7 @@ impl< ), ).into()); } - if let Some(output) = &output { - anyhow::ensure!( - output.realm_sub_id == scheduled_proposer, - "finalize output sub_id is not the scheduled proposer" - ); - let proposer_user_id = user_ids - .iter() - .find(|(sub_id, _)| *sub_id == proposal.proposer_sub_id) - .map(|(_, user_id)| *user_id) - .ok_or_else(|| anyhow::anyhow!("GUTA proposer sub_id {} is not a validator", proposal.proposer_sub_id))?; - anyhow::ensure!( - output.validator_user_id == N::F::from_u64_value(proposer_user_id), - "finalize output fee user is not the scheduled proposer's validator leaf user" - ); - } - // 6. Certificate: checkpoint-tree keys, replication wait and mandatory - // scheduled-proposer inclusion. Its aggregate BLS signature over the - // proposal identity D is the finalizer authorization. validate_certificate(&proposal, &certificate, &validator_sub_ids, &keys).map_err(|error| { GutaSubmitError::illegal( RealmFinalizeSubmitCode::InvalidCertificate, @@ -996,7 +877,7 @@ impl< ) })?; anyhow::ensure!( - votes_meet_wait(validator_sub_ids.len(), proposal.proposer_sub_id, &certificate.signer_sub_ids()), + votes_meet_wait(validator_sub_ids.len(), &certificate.signer_sub_ids()), "GUTA Certificate for realm {realm_id} below replication wait" ); anyhow::ensure!( @@ -1005,9 +886,8 @@ impl< proposal.proposer_sub_id ); tracing::info!( - "realm P2P certificate admitted realm={} inclusion_checkpoint_id={} proposal={} proposer_sub_id={} signers={} signer_sub_ids={:?}", + "realm P2P certificate admitted realm={} proposal={} proposer_sub_id={} signers={} signer_sub_ids={:?}", realm_id, - inclusion_checkpoint_id, hex::encode(proposal.proposal_id), proposal.proposer_sub_id, certificate.popcount(), @@ -1022,45 +902,23 @@ impl< mod tests { use super::*; - /// §5: external Realm admission is type 63 only. A non-63 submission with - /// an empty binding must be rejected, not silently admitted (this is the - /// path that previously returned Ok without any checks at all). #[test] fn admission_rejects_non_finalize_type_with_empty_binding() { - let error = validate_external_guta_admission( - ProvingJobCircuitType::GUTATwoGUTALinear as u32, - &[], - ) - .unwrap_err(); + let error = validate_external_guta_admission(ProvingJobCircuitType::GUTATwoGUTALinear as u32) + .unwrap_err(); assert!(error.to_string().contains("requires job type 63")); } - /// A non-63 submission cannot launder itself by supplying a binding: the - /// type check fires first. #[test] fn admission_rejects_non_finalize_type_even_with_binding() { - let error = validate_external_guta_admission( - ProvingJobCircuitType::GUTATwoGUTALinear as u32, - &[0u8; 442], - ) - .unwrap_err(); + let error = validate_external_guta_admission(ProvingJobCircuitType::GUTATwoGUTALinear as u32) + .unwrap_err(); assert!(error.to_string().contains("requires job type 63")); } - /// Type 63 without a binding is the flagged empty-binding admission hole. - #[test] - fn admission_rejects_finalize_type_with_empty_binding() { - let error = - validate_external_guta_admission(ProvingJobCircuitType::RealmFinalizeGUTA as u32, &[]) - .unwrap_err(); - assert!(error.to_string().contains("requires a finalize binding")); - } - - /// The legitimate shape passes the shape check and proceeds to the - /// certificate/binding verification stages. #[test] fn admission_accepts_finalize_type_with_binding() { - validate_external_guta_admission(ProvingJobCircuitType::RealmFinalizeGUTA as u32, &[0u8; 442]) - .expect("type 63 with binding is admissible"); + validate_external_guta_admission(ProvingJobCircuitType::RealmFinalizeGUTA as u32) + .expect("type 63 is admissible"); } } diff --git a/psy_node_common/src/guta_planner/realm_guta_planner.rs b/psy_node_common/src/guta_planner/realm_guta_planner.rs index 7a97b3db..fd2e30f1 100644 --- a/psy_node_common/src/guta_planner/realm_guta_planner.rs +++ b/psy_node_common/src/guta_planner/realm_guta_planner.rs @@ -27,6 +27,7 @@ use psy_data::{ queue_items::realm_user_update::PsyRealmUserUpdateQueueItem, v1::qdata::{ checkpoint::{PQEDCheckpointLeaf, PQEDCheckpointLeafCompactWithStateRoots}, + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, user::PQEDUserLeaf, }, worker::{ @@ -93,6 +94,7 @@ pub struct RealmGUTAPlanner { pub user_contract_tree_updates_ffs: Vec, pub contract_state_tree_updates_ffs: Vec, pub user_leaf_updates_ffs: Vec, + user_leaf_update_offsets: HashMap, /// IMT (Indexed Merkle Tree) leaf preimage data for contract state trees. /// Accumulated from end cap submissions. pub contract_state_imt_leaves_ffs: Vec, @@ -141,6 +143,7 @@ impl RealmGUTAPlanner { user_contract_tree_updates_ffs: Vec::new(), contract_state_tree_updates_ffs: Vec::new(), user_leaf_updates_ffs: Vec::new(), + user_leaf_update_offsets: HashMap::new(), contract_state_imt_leaves_ffs: Vec::new(), current_checkpoint_root, current_checkpoint_id, @@ -162,7 +165,6 @@ impl RealmGUTAPlanner { } } - pub fn set_validator_proofs(&mut self, proofs: RealmGUTAValidatorProofs) -> anyhow::Result<()> where F: ToU64Value, @@ -179,6 +181,34 @@ impl RealmGUTAPlanner { impl> RealmGUTAPlanner { + fn update_user_leaf(&mut self, user_leaf: &PQEDUserLeaf) -> anyhow::Result<()> { + let user_id = user_leaf.user_id.to_u64_value(); + let user_leaf_bytes = user_leaf.psy_ser_to_bytes_vec()?; + anyhow::ensure!( + user_leaf_bytes.len() == PSY_OBJECT_FFS_SIZE_USER_LEAF, + "User leaf FFS record for user {} is {} bytes, expected {}", + user_id, + user_leaf_bytes.len(), + PSY_OBJECT_FFS_SIZE_USER_LEAF + ); + if let Some(offset) = self.user_leaf_update_offsets.get(&user_id).copied() { + let end = offset + .checked_add(PSY_OBJECT_FFS_SIZE_USER_LEAF) + .ok_or_else(|| anyhow::anyhow!("User leaf update offset overflow for user {}", user_id))?; + anyhow::ensure!( + end <= self.user_leaf_updates_ffs.len(), + "User leaf update offset is out of bounds for user {}", + user_id + ); + self.user_leaf_updates_ffs[offset..end].copy_from_slice(&user_leaf_bytes); + } else { + let offset = self.user_leaf_updates_ffs.len(); + self.user_leaf_updates_ffs.extend_from_slice(&user_leaf_bytes); + self.user_leaf_update_offsets.insert(user_id, offset); + } + Ok(()) + } + pub async fn populate_future_end_cap_job>( _chain_id: u64, realm_identifier: &QRealmIdentifier, @@ -214,9 +244,9 @@ impl> RealmGUTAPlanner { realm_id_u64, realm_sub_id_u64, )?; - if single_header.checkpoint_id != queue_item.expected_fake_checkpoint_id { - tracing::info!("Skipping end-cap job population due to fake checkpoint ID mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", - queue_item.expected_fake_checkpoint_id, + if single_header.checkpoint_id != queue_item.submission_nonce { + tracing::info!("Skipping end-cap job population due to a submission_nonce mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", + queue_item.submission_nonce, single_header.checkpoint_id ); return Ok(None); @@ -392,9 +422,9 @@ impl> RealmGUTAPlanner { )?; tracing::debug!("Single header: {}", serde_json::to_string_pretty(&single_header)?); - if single_header.checkpoint_id != queue_item.expected_fake_checkpoint_id { - tracing::info!("Skipping end-cap job population due to fake checkpoint ID mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", - queue_item.expected_fake_checkpoint_id, + if single_header.checkpoint_id != queue_item.submission_nonce { + tracing::info!("Skipping end-cap job population due to a submission_nonce mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", + queue_item.submission_nonce, single_header.checkpoint_id ); return Ok(0); @@ -419,8 +449,7 @@ impl> RealmGUTAPlanner { self.user_contract_tree_updates_ffs.extend_from_slice(&single_payload); self.contract_state_tree_updates_ffs.extend_from_slice(&double_payload); - self.user_leaf_updates_ffs - .extend_from_slice(&queue_item.new_user_leaf.psy_ser_to_bytes_vec()?); + self.update_user_leaf(&queue_item.new_user_leaf)?; if self.current_validator_user_leaf.as_ref().map(|leaf| leaf.user_id.to_u64_value()) == Some(user_id) { self.current_validator_user_leaf = Some(queue_item.new_user_leaf.clone()); } @@ -610,9 +639,9 @@ impl> RealmGUTAPlanner { QBlobMerkleNodeTreeType::UserContractTree, )?; tracing::debug!("Single header: {}", serde_json::to_string_pretty(&single_header)?); - if single_header.checkpoint_id != queue_item.expected_fake_checkpoint_id { - tracing::info!("Skipping end-cap job population due to fake checkpoint ID mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", - queue_item.expected_fake_checkpoint_id, + if single_header.checkpoint_id != queue_item.submission_nonce { + tracing::info!("Skipping end-cap job population due to a submission_nonce mismatch: expected {}, found {}. Likely got overwritten due to a race condition. Gracefully skipping.", + queue_item.submission_nonce, single_header.checkpoint_id ); return Ok(0); @@ -638,8 +667,7 @@ impl> RealmGUTAPlanner { self.user_contract_tree_updates_ffs.extend_from_slice(&single_payload); self.contract_state_tree_updates_ffs.extend_from_slice(&double_payload); - self.user_leaf_updates_ffs - .extend_from_slice(&queue_item.new_user_leaf.psy_ser_to_bytes_vec()?); + self.update_user_leaf(&queue_item.new_user_leaf)?; if self.current_validator_user_leaf.as_ref().map(|leaf| leaf.user_id.to_u64_value()) == Some(user_id) { self.current_validator_user_leaf = Some(queue_item.new_user_leaf.clone()); } @@ -1055,7 +1083,6 @@ impl> RealmGUTAPlanner { } let finalizer_bytes = finalizer_input.psy_ser_into_bytes_vec()?; - let new_user_leaf_bytes = new_user_leaf.psy_ser_to_bytes_vec()?; let new_total_jobs = self .total_jobs .checked_add(1) @@ -1076,7 +1103,7 @@ impl> RealmGUTAPlanner { self.planned_jobs[finalizer_level].push(finalizer_job); self.total_jobs = new_total_jobs; - self.user_leaf_updates_ffs.extend_from_slice(&new_user_leaf_bytes); + self.update_user_leaf(&new_user_leaf)?; self.current_validator_user_leaf = Some(new_user_leaf); let final_header = GlobalUserTreeAggregatorHeaderWithJobId { @@ -1187,7 +1214,10 @@ impl> RealmGUTAPlanner { update_user_contract_tree_nodes_ffs: std::mem::take(&mut self.user_contract_tree_updates_ffs), update_contract_state_tree_nodes_ffs: std::mem::take(&mut self.contract_state_tree_updates_ffs), update_contract_state_imt_leaves_ffs: std::mem::take(&mut self.contract_state_imt_leaves_ffs), - update_user_leaves_ffs: std::mem::take(&mut self.user_leaf_updates_ffs), + update_user_leaves_ffs: { + self.user_leaf_update_offsets.clear(); + std::mem::take(&mut self.user_leaf_updates_ffs) + }, guta_header: root_header, }, job_ids: std::mem::take(&mut self.planned_jobs).into_iter().filter(|jobs| !jobs.is_empty()).collect(), @@ -1217,7 +1247,10 @@ impl> RealmGUTAPlanner { update_user_contract_tree_nodes_ffs: std::mem::take(&mut self.user_contract_tree_updates_ffs), update_contract_state_tree_nodes_ffs: std::mem::take(&mut self.contract_state_tree_updates_ffs), update_contract_state_imt_leaves_ffs: std::mem::take(&mut self.contract_state_imt_leaves_ffs), - update_user_leaves_ffs: std::mem::take(&mut self.user_leaf_updates_ffs), + update_user_leaves_ffs: { + self.user_leaf_update_offsets.clear(); + std::mem::take(&mut self.user_leaf_updates_ffs) + }, guta_header: root_header, }, job_ids: std::mem::take(&mut self.planned_jobs).into_iter().filter(|x| !x.is_empty()).collect(), diff --git a/psy_node_common/src/guta_planner/realm_guta_planner_tests/test_env.rs b/psy_node_common/src/guta_planner/realm_guta_planner_tests/test_env.rs index 4789a515..5abafa55 100644 --- a/psy_node_common/src/guta_planner/realm_guta_planner_tests/test_env.rs +++ b/psy_node_common/src/guta_planner/realm_guta_planner_tests/test_env.rs @@ -21,6 +21,7 @@ use psy_data::{ queue_items::realm_user_update::PsyRealmUserUpdateQueueItem, v1::qdata::{ contract::{DashMapContractHeightCache, PSimpleContractHeightCache}, + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, public_key::PZKPublicKeyInfo, user::PQEDUserLeaf, user_end_cap_result::PUPSEndCapResultCompact, @@ -505,7 +506,7 @@ impl RGPTestChainState { let queue_item = PsyRealmUserUpdateQueueItem { job_id: job_id, - expected_fake_checkpoint_id: fake_checkpoint_id, + submission_nonce: fake_checkpoint_id, old_user_leaf_hash: old_leaf_hash, new_user_leaf_hash, new_user_leaf, @@ -1284,6 +1285,48 @@ async fn test_finalizer_single_endcap_dispatch_and_fee_credit() -> anyhow::Resul Ok(()) } +#[tokio::test] +async fn validator_fee_update_emits_one_leaf_record_for_checkpoint() -> anyhow::Result<()> { + let mut state = RGPTestChainState::create_for_tests().await?; + state.add_new_contract(12).await?; + state.finalizer_user.user_leaf.nonce = F::from_u8_value(1); + state.first_realm_global_user_tree.set_leaf( + state.finalizer_user.user_id, + state.finalizer_user.user_leaf.qfhash::(), + ); + let validator_user = state.finalizer_user.clone(); + state.users.insert(validator_user.user_id, validator_user.clone()); + let transactions = state.gen_rand_contract_updates_for_ups(1, 1)?; + let output = state + .process_checkpoint(&[], &[(validator_user.user_id, transactions)], true) + .await? + .expect("validator EndCap must produce a Realm GUTA output"); + + let serialized_user_leaves = output.db_output.update_user_leaves_ffs.chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF); + assert!(serialized_user_leaves.remainder().is_empty()); + let validator_leaves = serialized_user_leaves + .map(PQEDUserLeaf::::psy_ser_from_slice) + .collect::>>()? + .into_iter() + .filter(|leaf| leaf.user_id.to_u64_value() == validator_user.user_id) + .collect::>(); + assert_eq!(validator_leaves.len(), 1); + let fee = output.db_output.guta_header.header.stats.da_fees_collected; + assert_eq!(validator_leaves[0].balance, validator_user.user_leaf.balance + fee); + let planned_checkpoint_id = state.checkpoint_id - 1; + if fee != F::ZERO_VALUE { + assert_eq!( + validator_leaves[0].last_checkpoint_id, + F::from_u64_value(planned_checkpoint_id) + ); + } + assert_eq!( + state.db.get_user_leaf(state.checkpoint_id, validator_user.user_id).await?, + validator_leaves[0] + ); + Ok(()) +} + #[tokio::test] async fn test_finalizer_missing_validator_proofs_fails_closed() -> anyhow::Result<()> { let mut state = RGPTestChainState::create_for_tests().await?; @@ -1310,7 +1353,7 @@ async fn deferred_endcap_conflict_preserves_authoritative_leaf() -> anyhow::Resu let mut leaf = state.finalizer_user.user_leaf.clone(); leaf.balance += F::from_u64_value(9); let item = PsyRealmUserUpdateQueueItem { - job_id: QProvingJobDataID::new_invalid_job_id(), expected_fake_checkpoint_id: 0, + job_id: QProvingJobDataID::new_invalid_job_id(), submission_nonce: 0, old_user_leaf_hash: Hash::from_u64x4([999, 0, 0, 0]), new_user_leaf_hash: leaf.qfhash::(), events: vec![], new_user_leaf: leaf, stats: GUTAStats::get_zero_value(), @@ -1642,7 +1685,7 @@ mod realm_gatherer_fixture_tests { // validation: the taken job must land back in the shared slot. let job = PlannedFutureEndCapJob { queue_item: PsyRealmUserUpdateQueueItem { - job_id: QProvingJobDataID::new_invalid_job_id(), expected_fake_checkpoint_id: 0, + job_id: QProvingJobDataID::new_invalid_job_id(), submission_nonce: 0, old_user_leaf_hash: leaf_hash, new_user_leaf_hash: leaf_hash, events: vec![], new_user_leaf: validator_user.user_leaf.clone(), stats: GUTAStats::get_zero_value(), }, @@ -1670,7 +1713,7 @@ mod realm_gatherer_fixture_tests { leaf.last_checkpoint_id = F::from_u64_value(5); gatherer.guta_planner.future_pending_end_cap_jobs.push(PlannedFutureEndCapJob { queue_item: PsyRealmUserUpdateQueueItem { - job_id: QProvingJobDataID::new_invalid_job_id(), expected_fake_checkpoint_id: 0, + job_id: QProvingJobDataID::new_invalid_job_id(), submission_nonce: 0, old_user_leaf_hash: Hash::get_zero_value(), new_user_leaf_hash: leaf.qfhash::(), events: vec![], new_user_leaf: leaf, stats: GUTAStats::get_zero_value(), }, @@ -1692,7 +1735,7 @@ mod realm_gatherer_fixture_tests { leaf.last_checkpoint_id = F::from_u64_value(5); gatherer.guta_planner.future_pending_end_cap_jobs.push(PlannedFutureEndCapJob { queue_item: PsyRealmUserUpdateQueueItem { - job_id: QProvingJobDataID::new_invalid_job_id(), expected_fake_checkpoint_id: 0, + job_id: QProvingJobDataID::new_invalid_job_id(), submission_nonce: 0, old_user_leaf_hash: Hash::get_zero_value(), new_user_leaf_hash: leaf.qfhash::(), events: vec![], new_user_leaf: leaf, stats: GUTAStats::get_zero_value(), }, diff --git a/psy_node_common/src/p2p/realm_coordinator.rs b/psy_node_common/src/p2p/realm_coordinator.rs index 05816389..ca6262c2 100644 --- a/psy_node_common/src/p2p/realm_coordinator.rs +++ b/psy_node_common/src/p2p/realm_coordinator.rs @@ -77,10 +77,9 @@ impl>, certificate: Option>, - finalize_binding: Vec, ) -> anyhow::Result<()> { self.client - .submit_guta(input, proof, realm_id, proposal, certificate, finalize_binding) + .submit_guta(input, proof, realm_id, proposal, certificate) .await .map_err(|error| match error { JsonRpseeClientError::Call(object) => GutaSubmitError::from_error_object(&object) diff --git a/psy_node_common/src/queue/gatherer.rs b/psy_node_common/src/queue/gatherer.rs index 6d7ad74e..d7e435f2 100644 --- a/psy_node_common/src/queue/gatherer.rs +++ b/psy_node_common/src/queue/gatherer.rs @@ -13,6 +13,10 @@ use crate::{ utils::processor_status::ProcessorStatus, }; +#[derive(Debug, thiserror::Error)] +#[error("GATHERER_{0}: gatherer channel closed")] +pub struct GathererChannelClosed(pub u32); + pub enum GathererTreeCommand { Finalize { reply: oneshot::Sender>, @@ -123,15 +127,15 @@ impl anyhow::Result<()> { self.qk.begin_shutdown()?; let (response_tx, response_rx) = oneshot::channel(); - self.trigger_tx.send(response_tx).await?; - let _result = response_rx.await??; + self.trigger_tx.send(response_tx).await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))?; + let _result = response_rx.await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))??; Ok(()) } pub async fn finalize_gathering_and_update_queue_key(&mut self, unique_id: u128) -> anyhow::Result { self.qk.set_unique_id(unique_id)?; let (response_tx, response_rx) = oneshot::channel(); - self.trigger_tx.send(response_tx).await?; - let result = response_rx.await??; + self.trigger_tx.send(response_tx).await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))?; + let result = response_rx.await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))??; Ok(result) } } @@ -241,8 +245,8 @@ impl anyhow::Result { self.qk.set_unique_id(unique_id)?; let (response_tx, response_rx) = oneshot::channel(); - if response_rx.is_terminated() { - anyhow::bail!("GATHERER_{QUEUE_TOPIC_ID}: Response channel was terminated before sending."); - } else if response_tx.is_closed() { - anyhow::bail!("GATHERER_{QUEUE_TOPIC_ID}: Response channel was closed before sending."); - } tracing::info!("start finish finalize_gathering_and_update_queue_key for GATHERER_{QUEUE_TOPIC_ID}"); self.trigger_tx .send(GathererTreeCommand::Finalize { reply: response_tx }) - .await?; - let result = response_rx.await??; + .await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))?; + let result = response_rx.await.map_err(|_| GathererChannelClosed(QUEUE_TOPIC_ID))??; tracing::info!("end finish finalize_gathering_and_update_queue_key for GATHERER_{QUEUE_TOPIC_ID}"); Ok(result) } @@ -273,8 +272,8 @@ impl { + tracing::error!("GATHERER_{QUEUE_TOPIC_ID}: bootstrap failed: {error:#}"); if let Some(command) = command { match command { GathererTreeCommand::Finalize { reply } | GathererTreeCommand::Stop { reply } => { @@ -577,3 +577,95 @@ pub async fn gatherer_runner_for_tree< tracing::info!("GATHERER_{QUEUE_TOPIC_ID}: Handoff complete. Cycle restarting."); } } + +#[cfg(test)] +mod tests { + use super::*; + use parth_core::{data::queue::queue_key::QPBaseQueueType, pgoldilocks::PoseidonHasher, PHash}; + use psy_data::v1::qdata::public_key::PZKPublicKeyInfo; + use psy_node_store_memory::temp_store::InMemoryTempStore; + + type Item = PZKPublicKeyInfo; + type Tree = SimpleMemoryMerkleRecorderStore; + type Gatherer = EphemeralQueueGathererWithTree<32, Item, ()>; + + fn queue_key() -> QPStandardUniqueIdQueueKey<32, Item> { + QPStandardUniqueIdQueueKey { + realm_id: 0, realm_sub_id: 1, unique_id: 0, task_group: 0, + queue_type: QPBaseQueueType::StandardEphemeral, + _phantom_queue_item: std::marker::PhantomData, + } + } + + #[derive(Debug, thiserror::Error)] + #[error("bootstrap proof mismatch")] + struct BootstrapFailure; + + struct FailingBuilder; + + #[async_trait::async_trait] + impl QueueGathererItemBuilderWithTree<(Arc, Arc), Tree> for FailingBuilder { + type Output = (); + + async fn create_new_with_tree(_: &mut Tree, _: u128, (entered, release): (Arc, Arc)) -> anyhow::Result { + entered.notify_one(); + release.notified().await; + Err(anyhow::Error::new(BootstrapFailure).context("restoring gatherer")) + } + + async fn update_from_queue_item_with_tree(&mut self, _: &mut Tree, _: Vec) -> anyhow::Result<()> { unreachable!() } + async fn update_from_many_queue_items_with_tree(&mut self, _: &mut Tree, _: Vec>) -> anyhow::Result<()> { unreachable!() } + async fn finalize_with_tree(self, _: &mut Tree) -> anyhow::Result<()> { unreachable!() } + } + + #[tokio::test] + async fn queued_finalize_recovers_original_bootstrap_error_from_owner() { + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let tree = Arc::new(tokio::sync::RwLock::new(Tree::new(4))); + let status = ProcessorStatus::new(); + status.mark_running(); + let (mut gatherer, handle) = Gatherer::new_with_status_shared::<_, _, PHash, PoseidonHasher, FailingBuilder>( + Arc::new(InMemoryTempStore::new("queued-bootstrap-test".to_string(), 1, 0)), + (entered.clone(), release.clone()), queue_key(), tree, status, + ); + entered.notified().await; + let mut finalize = Box::pin(gatherer.finalize_gathering_and_update_queue_key(1)); + tokio::select! { + biased; + result = &mut finalize => panic!("finalize completed before bootstrap: {result:?}"), + _ = std::future::ready(()) => {} + } + release.notify_one(); + let closed = finalize.await.unwrap_err(); + assert_eq!(closed.downcast_ref::().unwrap().0, 32); + let original = handle.await.unwrap().unwrap_err(); + assert!(original.downcast_ref::().is_some()); + } + + #[tokio::test] + async fn closed_trigger_is_typed_for_finalize_stop_and_fast_forward() { + let (trigger_tx, trigger_rx) = mpsc::channel(1); + let mut gatherer = Gatherer { qk: QueueKeyStatusManager::new(queue_key()), trigger_tx }; + drop(trigger_rx); + let finalize = gatherer.finalize_gathering_and_update_queue_key(1).await.unwrap_err(); + let fast_forward = gatherer.fast_forward(Vec::new()).await.unwrap_err(); + let stop = gatherer.stop_gracefully().await.unwrap_err(); + for error in [finalize, fast_forward, stop] { + assert_eq!(error.downcast_ref::().unwrap().0, 32); + } + } + + #[tokio::test] + async fn finalize_preserves_inner_error_identity() { + let (trigger_tx, mut trigger_rx) = mpsc::channel(1); + let mut gatherer = Gatherer { qk: QueueKeyStatusManager::new(queue_key()), trigger_tx }; + let owner = tokio::spawn(async move { + let Some(GathererTreeCommand::Finalize { reply }) = trigger_rx.recv().await else { panic!("expected finalize") }; + reply.send(Err(anyhow::Error::new(BootstrapFailure))).unwrap(); + }); + let error = gatherer.finalize_gathering_and_update_queue_key(1).await.unwrap_err(); + assert!(error.downcast_ref::().is_some()); + owner.await.unwrap(); + } +} diff --git a/psy_node_common/src/realm/edge/handler.rs b/psy_node_common/src/realm/edge/handler.rs index 06807d69..199ec36d 100644 --- a/psy_node_common/src/realm/edge/handler.rs +++ b/psy_node_common/src/realm/edge/handler.rs @@ -311,7 +311,7 @@ impl< .db_reader .get_checkpoint_global_state_roots(base_checkpoint_id) .await?; - let (validator_sub_ids, _, _) = load_realm_validators_from_tree::( + let (validator_sub_ids, _, _, _) = load_realm_validators_from_tree::( self.db_reader.as_ref(), self.chain_id, base_checkpoint_id, @@ -1059,7 +1059,7 @@ impl< let queue_item = PsyRealmUserUpdateQueueItem { job_id: job_id, - expected_fake_checkpoint_id: submission_token, + submission_nonce: submission_token, old_user_leaf_hash: old_leaf_hash, new_user_leaf_hash, new_user_leaf, diff --git a/psy_node_common/src/realm/network/behaviour.rs b/psy_node_common/src/realm/network/behaviour.rs index e5135a55..0d1b1616 100644 --- a/psy_node_common/src/realm/network/behaviour.rs +++ b/psy_node_common/src/realm/network/behaviour.rs @@ -11,7 +11,8 @@ //! does not include `kad` or `upnp`, and the slim port does not require them. use crate::realm::network::codec::{ - DirectBodyCodec, EndCapForwardCodec, DIRECT_BODY_PROTOCOL_ID, END_CAP_FORWARD_PROTOCOL_ID, + BodyChunkCodec, EndCapForwardCodec, ProposalLookupCodec, DIRECT_BODY_PROTOCOL_ID, + END_CAP_FORWARD_PROTOCOL_ID, PROPOSAL_LOOKUP_PROTOCOL_ID, }; use crate::realm::network::config::RealmNetworkConfig; use libp2p::autonat; @@ -41,7 +42,8 @@ pub fn vote_topic(realm_id: u32) -> gossipsub::IdentTopic { pub struct RealmBehaviour { pub gossipsub: gossipsub::Behaviour, pub identify: identify::Behaviour, - pub direct_body: request_response::Behaviour, + pub body_chunk: request_response::Behaviour, + pub proposal_lookup: request_response::Behaviour, pub end_cap_forward: request_response::Behaviour, pub relay_client: relay::client::Behaviour, pub relay_server: Toggle, @@ -92,7 +94,7 @@ impl RealmBehaviour { .with_cache_size(256), ); - let direct_body_config = request_response::Config::default() + let body_chunk_config = request_response::Config::default() .with_request_timeout(Duration::from_secs(psy_data::p2p::DIRECT_REQUEST_TIMEOUT_SECS)) .with_max_concurrent_streams(MAX_CONCURRENT_DIRECT_EXCHANGES); let end_cap_config = request_response::Config::default() @@ -101,7 +103,7 @@ impl RealmBehaviour { // DirectBody is validator-only. EndCap forward is Edge-only: validators // and bootnodes neither advertise nor accept the forwarding protocol. - let direct_body_protocols: &[(StreamProtocol, request_response::ProtocolSupport)] = + let body_chunk_protocols: &[(StreamProtocol, request_response::ProtocolSupport)] = if is_bootnode || is_edge { &[] } else { @@ -120,8 +122,24 @@ impl RealmBehaviour { &[] }; - let direct_body = - request_response::Behaviour::new(direct_body_protocols.iter().cloned(), direct_body_config); + let proposal_lookup_config = request_response::Config::default() + .with_request_timeout(Duration::from_secs(psy_data::p2p::DIRECT_REQUEST_TIMEOUT_SECS)) + .with_max_concurrent_streams(MAX_CONCURRENT_DIRECT_EXCHANGES); + let proposal_lookup_protocols: &[(StreamProtocol, request_response::ProtocolSupport)] = + if is_bootnode || is_edge { + &[] + } else { + &[( + StreamProtocol::new(PROPOSAL_LOOKUP_PROTOCOL_ID), + request_response::ProtocolSupport::Full, + )] + }; + let body_chunk = + request_response::Behaviour::new(body_chunk_protocols.iter().cloned(), body_chunk_config); + let proposal_lookup = request_response::Behaviour::new( + proposal_lookup_protocols.iter().cloned(), + proposal_lookup_config, + ); let end_cap_forward = request_response::Behaviour::new( end_cap_protocols.iter().cloned(), end_cap_config, @@ -138,7 +156,8 @@ impl RealmBehaviour { Self { gossipsub, identify, - direct_body, + body_chunk, + proposal_lookup, end_cap_forward, relay_client, relay_server, @@ -148,8 +167,9 @@ impl RealmBehaviour { } } -/// Register a known validator peer for mesh membership. -pub fn add_known_address(behaviour: &mut RealmBehaviour, peer_id: PeerId, _address: libp2p::Multiaddr) { +/// Register a realm peer as a gossipsub explicit peer so proposal and vote +/// delivery never depends on mesh gossip luck. +pub fn register_explicit_peer(behaviour: &mut RealmBehaviour, peer_id: PeerId) { behaviour.gossipsub.add_explicit_peer(&peer_id); } diff --git a/psy_node_common/src/realm/network/codec.rs b/psy_node_common/src/realm/network/codec.rs index b3253678..4c767722 100644 --- a/psy_node_common/src/realm/network/codec.rs +++ b/psy_node_common/src/realm/network/codec.rs @@ -1,8 +1,9 @@ //! libp2p `request_response::Codec` implementations for the slim Realm P2P //! protocols. //! -//! Two request/response protocols are wired: +//! Three request/response protocols are wired: //! - `/psy/realm/proposal-body/1` — bounded proposal body range exchange. +//! - `/psy/realm/proposal-lookup/1` — windowed root-pair proposal lookup. //! - `/psy/realm/end-cap-forward/2` — EndCap forward stream (56-byte header //! followed by `end_cap_input_len` input bytes and `proof_len` proof bytes); //! version 2 carries the 18-byte typed rejection response. @@ -16,20 +17,23 @@ use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use libp2p::request_response::Codec; use libp2p::swarm::StreamProtocol; use psy_data::p2p::{ - DirectBodyRequest, DirectBodyResponse, EndCapForwardHeader, EndCapForwardResponse, - ProtocolEncode, DIRECT_BODY_REQUEST_WIRE_BYTES, DIRECT_REQUEST_MAX_BYTES, - END_CAP_FORWARD_HEADER_WIRE_BYTES, END_CAP_FORWARD_RESPONSE_WIRE_BYTES, - MAX_END_CAP_FORWARD_BYTES, + BodyChunkRequest, BodyChunkResponse, EndCapForwardHeader, EndCapForwardResponse, + ProposalLookupRequest, ProposalLookupResponse, ProtocolEncode, + BODY_CHUNK_REQUEST_WIRE_BYTES, BODY_CHUNK_MAX_BYTES, END_CAP_FORWARD_HEADER_WIRE_BYTES, + END_CAP_FORWARD_RESPONSE_WIRE_BYTES, MAX_END_CAP_FORWARD_BYTES, + MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, PROPOSAL_LOOKUP_MAX_PAIRS, + PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES, }; use std::{fmt, io}; pub const DIRECT_BODY_PROTOCOL_ID: &str = "/psy/realm/proposal-body/1"; +pub const PROPOSAL_LOOKUP_PROTOCOL_ID: &str = "/psy/realm/proposal-lookup/1"; /// Bumped from `.../end-cap-forward/1` when the response grew from a bare /// 1-byte bool to the typed 18-byte rejection, so mixed-version peers never /// misread each other's responses. pub const END_CAP_FORWARD_PROTOCOL_ID: &str = "/psy/realm/end-cap-forward/2"; -const DIRECT_BODY_RESPONSE_OVERHEAD: usize = 53; +const BODY_CHUNK_RESPONSE_OVERHEAD: usize = 53; // --------------------------------------------------------------------------- @@ -37,20 +41,20 @@ const DIRECT_BODY_RESPONSE_OVERHEAD: usize = 53; // --------------------------------------------------------------------------- #[derive(Clone, Default)] -pub struct DirectBodyCodec; +pub struct BodyChunkCodec; #[async_trait] -impl Codec for DirectBodyCodec { +impl Codec for BodyChunkCodec { type Protocol = StreamProtocol; - type Request = DirectBodyRequest; - type Response = DirectBodyResponse; + type Request = BodyChunkRequest; + type Response = BodyChunkResponse; async fn read_request(&mut self, _: &Self::Protocol, io: &mut T) -> io::Result where T: AsyncRead + Unpin + Send, { - let bytes = read_to_end_bounded(io, DIRECT_BODY_REQUEST_WIRE_BYTES).await?; - DirectBodyRequest::decode_exact(&bytes).map_err(invalid_data) + let bytes = read_to_end_bounded(io, BODY_CHUNK_REQUEST_WIRE_BYTES).await?; + BodyChunkRequest::decode_exact(&bytes).map_err(invalid_data) } async fn read_response( @@ -61,9 +65,9 @@ impl Codec for DirectBodyCodec { where T: AsyncRead + Unpin + Send, { - let max = DIRECT_BODY_RESPONSE_OVERHEAD + DIRECT_REQUEST_MAX_BYTES as usize; + let max = BODY_CHUNK_RESPONSE_OVERHEAD + BODY_CHUNK_MAX_BYTES as usize; let bytes = read_to_end_bounded(io, max).await?; - DirectBodyResponse::decode_exact(&bytes).map_err(invalid_data) + BodyChunkResponse::decode_exact(&bytes).map_err(invalid_data) } async fn write_request( @@ -76,7 +80,7 @@ impl Codec for DirectBodyCodec { T: AsyncWrite + Unpin + Send, { let bytes = request.protocol_encode_to_vec(); - if bytes.len() != DIRECT_BODY_REQUEST_WIRE_BYTES { + if bytes.len() != BODY_CHUNK_REQUEST_WIRE_BYTES { return Err(invalid_data("invalid direct-body request length")); } write_all_and_close(io, &bytes).await @@ -91,13 +95,95 @@ impl Codec for DirectBodyCodec { where T: AsyncWrite + Unpin + Send, { - if response.data.len() > DIRECT_REQUEST_MAX_BYTES as usize { + if response.data.len() > BODY_CHUNK_MAX_BYTES as usize { return Err(invalid_data("direct-body response exceeds maximum")); } write_all_and_close(io, &response.protocol_encode_to_vec()).await } } +#[derive(Clone, Default)] +pub struct ProposalLookupCodec; + +#[async_trait] +impl Codec for ProposalLookupCodec { + type Protocol = StreamProtocol; + type Request = ProposalLookupRequest; + type Response = ProposalLookupResponse; + + async fn read_request(&mut self, _: &Self::Protocol, io: &mut T) -> io::Result + where + T: AsyncRead + Unpin + Send, + { + let bytes = read_to_end_bounded(io, PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES).await?; + ProposalLookupRequest::decode_exact(&bytes).map_err(invalid_data) + } + + async fn read_response( + &mut self, + _: &Self::Protocol, + io: &mut T, + ) -> io::Result + where + T: AsyncRead + Unpin + Send, + { + let bytes = read_to_end_bounded(io, MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES).await?; + ProposalLookupResponse::decode_exact(&bytes).map_err(invalid_data) + } + + async fn write_request( + &mut self, + _: &Self::Protocol, + io: &mut T, + request: Self::Request, + ) -> io::Result<()> + where + T: AsyncWrite + Unpin + Send, + { + if request.pairs.is_empty() || request.pairs.len() > PROPOSAL_LOOKUP_MAX_PAIRS { + return Err(invalid_data("invalid ProposalLookup pair count")); + } + let expected = 16 + request.pairs.len() * 64; + let bytes = request.protocol_encode_to_vec(); + if bytes.len() != expected { + return Err(invalid_data("invalid ProposalLookup request length")); + } + write_all_and_close(io, &bytes).await + } + + async fn write_response( + &mut self, + _: &Self::Protocol, + io: &mut T, + response: Self::Response, + ) -> io::Result<()> + where + T: AsyncWrite + Unpin + Send, + { + let has_candidates = response.entries.iter().any(|entry| !entry.candidates.is_empty()); + if response.entries.iter().any(|entry| { + entry.candidates.len() > psy_data::p2p::PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR + }) || (response.status == psy_data::p2p::ProposalLookupStatus::Candidates && !has_candidates) + || (response.status == psy_data::p2p::ProposalLookupStatus::Empty && has_candidates) + { + return Err(invalid_data("invalid ProposalLookup response")); + } + let mut expected = 5; + for entry in response.entries.iter().take(PROPOSAL_LOOKUP_MAX_PAIRS) { + let entry_bytes = 65 + entry.candidates.len() * psy_data::p2p::PROPOSAL_WIRE_BYTES; + if expected + entry_bytes > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { + break; + } + expected += entry_bytes; + } + let bytes = response.protocol_encode_to_vec(); + if bytes.len() != expected { + return Err(invalid_data("invalid ProposalLookup response length")); + } + write_all_and_close(io, &bytes).await + } +} + // --------------------------------------------------------------------------- // EndCap forward // --------------------------------------------------------------------------- diff --git a/psy_node_common/src/realm/network/drive.rs b/psy_node_common/src/realm/network/drive.rs index 3529922e..74892b64 100644 --- a/psy_node_common/src/realm/network/drive.rs +++ b/psy_node_common/src/realm/network/drive.rs @@ -1,9 +1,9 @@ //! Drive loop for [`RealmNetwork`]: commands, swarm events, vote waiters. -use crate::realm::network::behaviour::{proposal_topic, vote_topic, RealmBehaviourEvent}; +use crate::realm::network::behaviour::{proposal_topic, register_explicit_peer, vote_topic, RealmBehaviourEvent}; use crate::realm::network::codec::EndCapForwardRequest; use crate::realm::network::{ InsertOutcome, NetworkError, RealmNetwork, RealmNetworkCommand, RealmNetworkEvent, - StartOutcome, + ReassemblyBook, StartOutcome, }; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; @@ -11,20 +11,25 @@ use futures::{FutureExt, StreamExt}; use libp2p::gossipsub; use libp2p::multiaddr::Protocol; use libp2p::request_response::{ - self, InboundRequestId, OutboundRequestId, ResponseChannel, + self, OutboundRequestId, ResponseChannel, }; use libp2p::swarm::SwarmEvent; use libp2p::{Multiaddr, PeerId}; use psy_data::p2p::{ - vote_message, BlsPublicKey, DirectBodyRequest, DirectBodyResponse, EndCapForwardResponse, - EndCapRejectReason, NodeId, Proposal, ProposalPart, ProtocolEncode, Vote, - DIRECT_REQUEST_MAX_BYTES, MAINTENANCE_TICK_SECS, MAX_PROPOSAL_CHUNK_BYTES, - RANGE_REQUEST_RETRY_INTERVAL_SECS, + vote_message, BlsPublicKey, BodyChunkRequest, BodyChunkResponse, EndCapForwardResponse, + EndCapRejectReason, NodeId, Proposal, ProposalLookupRequest, ProposalLookupResponse, + ProposalPart, ProtocolEncode, ValidatorLeaf, Vote, BODY_CHUNK_MAX_BYTES, + MAINTENANCE_TICK_SECS, MAX_CONCURRENT_DIRECT_EXCHANGES, MAX_IN_FLIGHT_PROPOSALS, + MAX_PROPOSAL_CHUNK_BYTES, MAX_VALIDATORS_PER_REALM, MAX_VOTE_AUTH, + RANGE_REQUEST_RETRY_INTERVAL_SECS, VOTE_AUTH_TTL_SECS, }; use std::collections::{HashMap, HashSet}; use std::time::{Duration, Instant}; use tokio::sync::oneshot; +/// Bounded reconnect backoff for disconnected realm peers. +const REALM_PEER_REDIAL_BACKOFF_SECS: u64 = 15; + struct VoteWaiter { proposal_id: [u8; 32], threshold: usize, @@ -37,41 +42,67 @@ struct VoteWaiter { struct ProposalVoteAuth { vote_message: Vec, leaf_bls_keys: Vec<(u16, BlsPublicKey)>, + inserted_at: Instant, } struct DriveState { - published_bodies: HashMap<[u8; 32], (Proposal, Vec)>, proposal_source: HashMap<[u8; 32], PeerId>, end_cap_responses: HashMap< OutboundRequestId, oneshot::Sender>, >, - inbound_body: HashMap>, + pending_lookup: HashMap< + OutboundRequestId, + oneshot::Sender>, + >, + pending_body: HashMap< + OutboundRequestId, + oneshot::Sender>, + >, pending_direct: HashMap, vote_auth: HashMap<[u8; 32], ProposalVoteAuth>, vote_backlog: HashMap<[u8; 32], Vec>, vote_waiters: Vec, + validator_leaves: HashMap, end_cap_replies: FuturesUnordered< BoxFuture<'static, (ResponseChannel, Option)>, >, + lookup_replies: FuturesUnordered< + BoxFuture<'static, (ResponseChannel, Option)>, + >, + body_replies: FuturesUnordered< + BoxFuture<'static, (ResponseChannel, Option)>, + >, + connected_peers: HashSet, + explicit_peers: HashSet, + dial_attempts: HashMap, + learned_addresses: HashMap, } impl DriveState { fn new() -> Self { Self { - published_bodies: HashMap::new(), proposal_source: HashMap::new(), end_cap_responses: HashMap::new(), - inbound_body: HashMap::new(), + pending_lookup: HashMap::new(), + pending_body: HashMap::new(), pending_direct: HashMap::new(), vote_auth: HashMap::new(), vote_backlog: HashMap::new(), vote_waiters: Vec::new(), + validator_leaves: HashMap::new(), end_cap_replies: FuturesUnordered::new(), + lookup_replies: FuturesUnordered::new(), + body_replies: FuturesUnordered::new(), + connected_peers: HashSet::new(), + explicit_peers: HashSet::new(), + dial_attempts: HashMap::new(), + learned_addresses: HashMap::new(), } } } + pub async fn run_realm_network(mut network: RealmNetwork) { network.run().await; } @@ -79,20 +110,9 @@ pub async fn run_realm_network(mut network: RealmNetwork) { impl RealmNetwork { pub async fn run(&mut self) { let mut state = DriveState::new(); - for (peer_id, address) in self.config.bootnode_addresses.clone() { - let mut dial = address.clone(); - if !dial.iter().any(|p| matches!(p, Protocol::P2p(_))) { - dial.push(Protocol::P2p(peer_id)); - } - if let Err(error) = self.swarm.dial(dial) { - tracing::warn!( - realm_id = self.realm_id, - %peer_id, - %error, - "failed to dial Realm bootnode" - ); - } - } + state.validator_leaves.clone_from(&self.validator_leaves); + self.register_realm_peers(&mut state); + self.redial_realm_peers(&mut state); let mut tick = tokio::time::interval(Duration::from_secs(MAINTENANCE_TICK_SECS)); loop { @@ -122,6 +142,24 @@ impl RealmNetwork { ); } } + Some((channel, reply)) = state.lookup_replies.next() => { + if let Some(response) = reply { + let _ = self + .swarm + .behaviour_mut() + .proposal_lookup + .send_response(channel, response); + } + } + Some((channel, reply)) = state.body_replies.next() => { + if let Some(response) = reply { + let _ = self + .swarm + .behaviour_mut() + .body_chunk + .send_response(channel, response); + } + } _ = tick.tick() => { self.maintain(&mut state); } @@ -158,18 +196,14 @@ impl RealmNetwork { leaf_bls_keys, response, } => { - register_proposal_vote_auth(state, &proposal, leaf_bls_keys); + register_proposal_vote_auth(state, &proposal, leaf_bls_keys, Instant::now()); let result = publish_proposal_parts( &mut self.swarm, self.realm_id, &proposal, &body, ); - if result.is_ok() { - state - .published_bodies - .insert(proposal.proposal_id, (proposal, body)); - } else { + if result.is_err() { state.vote_auth.remove(&proposal.proposal_id); state.vote_backlog.remove(&proposal.proposal_id); } @@ -193,23 +227,37 @@ impl RealmNetwork { } let _ = response.send(result); } - RealmNetworkCommand::ServeBody { - request_id, - response_body, + RealmNetworkCommand::LookupProposal { + destination, + request, response, } => { - let result = match state.inbound_body.remove(&request_id) { - Some(channel) => self - .swarm - .behaviour_mut() - .direct_body - .send_response(channel, response_body) - .map_err(|_| NetworkError::DirectRequest("direct-body response channel closed".into())), - None => Err(NetworkError::DirectRequest( - "unknown inbound direct-body request".into(), - )), - }; - let _ = response.send(result); + if !has_validator_leaf(state, &destination) { + let _ = response.send(Err(NetworkError::NotAValidator(destination))); + return; + } + let request_id = self + .swarm + .behaviour_mut() + .proposal_lookup + .send_request(&destination.to_peer_id(), request); + state.pending_lookup.insert(request_id, response); + } + RealmNetworkCommand::RequestBody { + destination, + request, + response, + } => { + if !has_validator_leaf(state, &destination) { + let _ = response.send(Err(NetworkError::NotAValidator(destination))); + return; + } + let request_id = self + .swarm + .behaviour_mut() + .body_chunk + .send_request(&destination.to_peer_id(), request); + state.pending_body.insert(request_id, response); } RealmNetworkCommand::WaitVotes { proposal_id, @@ -243,6 +291,14 @@ impl RealmNetwork { response, }); } + RealmNetworkCommand::SetValidatorLeaves { leaves, response } => { + let result = replace_validator_leaves(state, leaves); + if result.is_ok() { + self.validator_leaves.clone_from(&state.validator_leaves); + self.register_realm_peers(state); + } + let _ = response.send(result); + } } } @@ -303,10 +359,57 @@ impl RealmNetwork { SwarmEvent::Behaviour(RealmBehaviourEvent::EndCapForward(event)) => { self.handle_end_cap_event(event, state); } - SwarmEvent::Behaviour(RealmBehaviourEvent::DirectBody(event)) => { - self.handle_direct_body_event(event, state); + SwarmEvent::Behaviour(RealmBehaviourEvent::BodyChunk(event)) => { + self.handle_body_chunk_event(event, state); + } + SwarmEvent::Behaviour(RealmBehaviourEvent::ProposalLookup(event)) => { + self.handle_proposal_lookup_event(event, state); + } + SwarmEvent::Behaviour(RealmBehaviourEvent::Gossipsub(gossipsub::Event::Subscribed { + peer_id, + topic, + })) => { + tracing::debug!(realm_id = self.realm_id, %peer_id, %topic, "Realm gossipsub peer subscribed"); + } + SwarmEvent::Behaviour(RealmBehaviourEvent::Gossipsub(gossipsub::Event::Unsubscribed { + peer_id, + topic, + })) => { + tracing::debug!(realm_id = self.realm_id, %peer_id, %topic, "Realm gossipsub peer unsubscribed"); } SwarmEvent::Behaviour(_) => {} + SwarmEvent::ConnectionEstablished { + peer_id, + endpoint, + .. + } => { + state.connected_peers.insert(peer_id); + state.dial_attempts.remove(&peer_id); + if self.is_realm_peer(state, &peer_id) { + register_explicit_peer(self.swarm.behaviour_mut(), peer_id); + if let libp2p::swarm::derive_prelude::ConnectedPoint::Dialer { address, .. } = endpoint { + state.learned_addresses.insert(peer_id, address.clone()); + } + } + tracing::debug!(realm_id = self.realm_id, %peer_id, "Realm P2P connection established"); + } + SwarmEvent::ConnectionClosed { + peer_id, + num_established, + cause, + .. + } => { + if num_established == 0 { + state.connected_peers.remove(&peer_id); + } + tracing::debug!( + realm_id = self.realm_id, + %peer_id, + remaining = num_established, + cause = ?cause, + "Realm P2P connection closed" + ); + } SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => { tracing::debug!(realm_id = self.realm_id, ?peer_id, %error, "outgoing connection error"); } @@ -335,14 +438,27 @@ impl RealmNetwork { total_parts, body_len, } => { - state - .proposal_source - .insert(proposal.proposal_id, source_peer); - match self - .reassembly - .start(proposal, total_parts, body_len, now) - { - Ok(StartOutcome::Inserted | StartOutcome::Duplicate) => {} + let proposal_id = proposal.proposal_id; + match admit_proposal_start( + state, + &mut self.reassembly, + proposal, + total_parts, + body_len, + source_peer, + now, + ) { + Ok(StartOutcome::Inserted) => { + tracing::info!( + realm_id = self.realm_id, + proposal = %hex::encode(proposal_id), + source = ?source, + total_parts, + body_len, + "Realm P2P proposal start accepted" + ); + } + Ok(StartOutcome::Duplicate) => {} Err(error) => { tracing::debug!(realm_id = self.realm_id, %error, "rejected ProposalPart::Start"); } @@ -361,6 +477,14 @@ impl RealmNetwork { Ok(InsertOutcome::Complete) => { match self.reassembly.finalize(&proposal_id) { Ok(complete) => { + retain_active_proposal_sources(state, &self.reassembly); + tracing::info!( + realm_id = self.realm_id, + proposal = %hex::encode(proposal_id), + source = ?source, + body_len = complete.body.len(), + "Realm P2P proposal body complete" + ); let _ = self.event_tx.try_send(RealmNetworkEvent::ProposalReady { source, proposal: complete.proposal, @@ -368,6 +492,7 @@ impl RealmNetwork { }); } Err(error) => { + retain_active_proposal_sources(state, &self.reassembly); tracing::debug!( realm_id = self.realm_id, %error, @@ -376,7 +501,18 @@ impl RealmNetwork { } } } - Ok(_) => {} + Ok(outcome) => { + if let Some(reassembly) = self.reassembly.get(&proposal_id) { + tracing::debug!( + realm_id = self.realm_id, + proposal = %hex::encode(proposal_id), + contiguous = reassembly.contiguous(), + body_len = reassembly.body_len(), + ?outcome, + "Realm P2P proposal chunk" + ); + } + } Err(error) => { tracing::debug!(realm_id = self.realm_id, %error, "rejected ProposalPart::Chunk"); } @@ -413,7 +549,9 @@ impl RealmNetwork { proof: request.proof, reply: reply_tx, }; - if self.event_tx.try_send(event).is_err() { + if reply_queue_at_capacity(state.end_cap_replies.len()) + || self.event_tx.try_send(event).is_err() + { let _ = self .swarm .behaviour_mut() @@ -448,45 +586,49 @@ impl RealmNetwork { } } - fn handle_direct_body_event( + fn handle_body_chunk_event( &mut self, - event: request_response::Event, + event: request_response::Event, state: &mut DriveState, ) { match event { - request_response::Event::Message { message, .. } => match message { + request_response::Event::Message { peer, message, .. } => match message { request_response::Message::Request { - request_id, request, channel, - } => match serve_published_body(&state.published_bodies, &request) { - Ok(response_body) => { - if self - .swarm - .behaviour_mut() - .direct_body - .send_response(channel, response_body) - .is_err() - { - tracing::debug!( - realm_id = self.realm_id, - "direct-body inbound response channel closed" - ); - } + .. + } => { + let Ok(source) = NodeId::from_peer_id(&peer) else { + return; + }; + if !has_validator_leaf(state, &source) { + return; } - Err(error) => { - state.inbound_body.insert(request_id, channel); - tracing::debug!( - realm_id = self.realm_id, - %error, - "no published body for inbound DirectBodyRequest" - ); + let (reply_tx, reply_rx) = oneshot::channel(); + if reply_queue_at_capacity(state.body_replies.len()) + || self + .event_tx + .try_send(RealmNetworkEvent::DirectBodyReceived { + source, + request, + reply: reply_tx, + }) + .is_err() + { + return; } - }, + state.body_replies.push( + async move { (channel, reply_rx.await.ok()) }.boxed(), + ); + } request_response::Message::Response { request_id, response, } => { + if let Some(tx) = state.pending_body.remove(&request_id) { + let _ = tx.send(Ok(response)); + return; + } let Some(proposal_id) = state.pending_direct.remove(&request_id) else { return; }; @@ -504,9 +646,11 @@ impl RealmNetwork { ) { Ok(InsertOutcome::Complete) => { if let Ok(complete) = self.reassembly.finalize(&proposal_id) { - let Some(source) = resolve_proposal_ready_source( + let source = resolve_proposal_ready_source( state.proposal_source.get(&proposal_id), - ) else { + ); + retain_active_proposal_sources(state, &self.reassembly); + let Some(source) = source else { tracing::warn!( realm_id = self.realm_id, "dropped ProposalReady: missing or invalid source NodeId" @@ -518,6 +662,8 @@ impl RealmNetwork { proposal: complete.proposal, body: complete.body, }); + } else { + retain_active_proposal_sources(state, &self.reassembly); } } Ok(_) => {} @@ -534,6 +680,10 @@ impl RealmNetwork { request_response::Event::OutboundFailure { request_id, error, .. } => { + if let Some(tx) = state.pending_body.remove(&request_id) { + let _ = tx.send(Err(NetworkError::DirectRequest(error.to_string()))); + return; + } if let Some(proposal_id) = state.pending_direct.remove(&request_id) { if let Some(reassembly) = self.reassembly.get_mut(&proposal_id) { reassembly.set_direct_request_active(false, Instant::now()); @@ -549,10 +699,67 @@ impl RealmNetwork { } } + fn handle_proposal_lookup_event( + &mut self, + event: request_response::Event, + state: &mut DriveState, + ) { + match event { + request_response::Event::Message { peer, message, .. } => match message { + request_response::Message::Request { + request, + channel, + .. + } => { + let Ok(source) = NodeId::from_peer_id(&peer) else { + return; + }; + if !has_validator_leaf(state, &source) { + return; + } + let (reply_tx, reply_rx) = oneshot::channel(); + if reply_queue_at_capacity(state.lookup_replies.len()) + || self + .event_tx + .try_send(RealmNetworkEvent::LookupReceived { + source, + request, + reply: reply_tx, + }) + .is_err() + { + return; + } + state.lookup_replies.push( + async move { (channel, reply_rx.await.ok()) }.boxed(), + ); + } + request_response::Message::Response { + request_id, + response, + } => { + if let Some(tx) = state.pending_lookup.remove(&request_id) { + let _ = tx.send(Ok(response)); + } + } + }, + request_response::Event::OutboundFailure { + request_id, error, .. + } => { + if let Some(tx) = state.pending_lookup.remove(&request_id) { + let _ = tx.send(Err(NetworkError::DirectRequest(error.to_string()))); + } + } + _ => {} + } + } + fn maintain(&mut self, state: &mut DriveState) { let now = Instant::now(); - let _ = self.reassembly.expire(now); + self.redial_realm_peers(state); + expire_reassembly(state, &mut self.reassembly, now); + expire_idle_vote_auth(state, now); let mut expired = Vec::new(); state.vote_waiters.retain_mut(|waiter| { @@ -592,8 +799,8 @@ impl RealmNetwork { if remaining == 0 { continue; } - let max_bytes = remaining.min(DIRECT_REQUEST_MAX_BYTES as u64) as u32; - let request = DirectBodyRequest { + let max_bytes = remaining.min(BODY_CHUNK_MAX_BYTES as u64) as u32; + let request = BodyChunkRequest { proposal_id, offset: reassembly.contiguous(), max_bytes, @@ -602,11 +809,102 @@ impl RealmNetwork { let request_id = self .swarm .behaviour_mut() - .direct_body + .body_chunk .send_request(&peer, request); state.pending_direct.insert(request_id, proposal_id); } } + + /// Validator and bootnode peers this node must stay connected to. + fn realm_peer_targets(&self, state: &DriveState) -> HashSet { + let mut targets: HashSet = state + .validator_leaves + .keys() + .map(|node_id| node_id.to_peer_id()) + .collect(); + targets.extend( + self.config + .bootnode_addresses + .iter() + .map(|(peer_id, _)| *peer_id), + ); + targets.remove(&self.swarm.local_peer_id()); + targets + } + + fn register_realm_peers(&mut self, state: &mut DriveState) { + let targets = self.realm_peer_targets(state); + if targets == state.explicit_peers { + return; + } + for peer_id in targets.difference(&state.explicit_peers) { + register_explicit_peer(self.swarm.behaviour_mut(), *peer_id); + } + for peer_id in state.explicit_peers.difference(&targets) { + self.swarm.behaviour_mut().gossipsub.remove_explicit_peer(peer_id); + } + tracing::info!( + realm_id = self.realm_id, + peers = ?targets, + "Realm gossipsub explicit peers registered" + ); + state.explicit_peers = targets; + } + + /// Reconnect disconnected realm peers with a bounded per-peer backoff. + fn redial_realm_peers(&mut self, state: &mut DriveState) { + let now = Instant::now(); + let backoff = Duration::from_secs(REALM_PEER_REDIAL_BACKOFF_SECS); + for peer_id in self.realm_peer_targets(state) { + if state.connected_peers.contains(&peer_id) { + continue; + } + if state + .dial_attempts + .get(&peer_id) + .is_some_and(|last| now.duration_since(*last) < backoff) + { + continue; + } + state.dial_attempts.insert(peer_id, now); + let dial: libp2p::swarm::dial_opts::DialOpts = + if let Some((_, address)) = self + .config + .bootnode_addresses + .iter() + .find(|(candidate, _)| *candidate == peer_id) + { + peer_dial_address(peer_id, address).into() + } else if let Some(address) = state.learned_addresses.get(&peer_id) { + peer_dial_address(peer_id, address).into() + } else { + peer_id.into() + }; + if let Err(error) = self.swarm.dial(dial) { + tracing::debug!(realm_id = self.realm_id, %peer_id, %error, "Realm peer dial failed"); + } + } + } + + fn is_realm_peer(&self, state: &DriveState, peer_id: &PeerId) -> bool { + state + .validator_leaves + .keys() + .any(|node_id| node_id.to_peer_id() == *peer_id) + || self + .config + .bootnode_addresses + .iter() + .any(|(candidate, _)| candidate == peer_id) + } +} + +fn peer_dial_address(peer_id: PeerId, address: &Multiaddr) -> Multiaddr { + let mut dial = address.clone(); + if !dial.iter().any(|p| matches!(p, Protocol::P2p(_))) { + dial.push(Protocol::P2p(peer_id)); + } + dial } fn publish_proposal_parts( @@ -650,43 +948,22 @@ fn publish_proposal_parts( Ok(()) } -fn serve_published_body( - published: &HashMap<[u8; 32], (Proposal, Vec)>, - request: &DirectBodyRequest, -) -> Result { - let (_proposal, body) = published.get(&request.proposal_id).ok_or_else(|| { - NetworkError::DirectRequest("unknown proposal body".into()) - })?; - let start = request.offset as usize; - if start > body.len() { - return Err(NetworkError::DirectRequest( - "direct-body offset past end".into(), - )); - } - let take = (request.max_bytes as usize).min(body.len() - start); - let data = body[start..start + take].to_vec(); - Ok(DirectBodyResponse { - offset: request.offset, - eof: start + take == body.len(), - body_len: body.len() as u64, - body_hash: sha_body_hash(body), - data, - }) -} - -fn sha_body_hash(body: &[u8]) -> [u8; 32] { - psy_data::p2p::sha256(body) -} - fn register_proposal_vote_auth( state: &mut DriveState, proposal: &Proposal, leaf_bls_keys: Vec<(u16, BlsPublicKey)>, + now: Instant, ) { let leaf_bls_keys = leaf_bls_keys .into_iter() .filter(|(sub_id, _)| *sub_id != proposal.proposer_sub_id) .collect(); + if !state.vote_auth.contains_key(&proposal.proposal_id) { + evict_idle_vote_auth(state); + if state.vote_auth.len() >= MAX_VOTE_AUTH { + return; + } + } state.vote_auth.insert( proposal.proposal_id, ProposalVoteAuth { @@ -697,10 +974,103 @@ fn register_proposal_vote_auth( &proposal.proposal_id, ), leaf_bls_keys, + inserted_at: now, }, ); } +fn evict_idle_vote_auth(state: &mut DriveState) { + while state.vote_auth.len() >= MAX_VOTE_AUTH { + let oldest_idle = state + .vote_auth + .iter() + .filter(|(proposal_id, _)| { + !state + .vote_waiters + .iter() + .any(|waiter| waiter.proposal_id == **proposal_id) + }) + .min_by_key(|(_, auth)| auth.inserted_at) + .map(|(proposal_id, _)| *proposal_id); + let Some(proposal_id) = oldest_idle else { + break; + }; + state.vote_auth.remove(&proposal_id); + state.vote_backlog.remove(&proposal_id); + } +} + +fn expire_idle_vote_auth(state: &mut DriveState, now: Instant) { + let ttl = Duration::from_secs(VOTE_AUTH_TTL_SECS); + let expired: Vec<[u8; 32]> = state + .vote_auth + .iter() + .filter(|(proposal_id, auth)| { + now.duration_since(auth.inserted_at) >= ttl + && !state + .vote_waiters + .iter() + .any(|waiter| waiter.proposal_id == **proposal_id) + }) + .map(|(proposal_id, _)| *proposal_id) + .collect(); + for proposal_id in expired { + state.vote_auth.remove(&proposal_id); + state.vote_backlog.remove(&proposal_id); + } +} + +fn admit_proposal_start( + state: &mut DriveState, + book: &mut ReassemblyBook, + proposal: Proposal, + total_parts: u32, + body_len: u64, + source_peer: PeerId, + now: Instant, +) -> Result { + let proposal_id = proposal.proposal_id; + let outcome = book.start(proposal, total_parts, body_len, now)?; + state.proposal_source.insert(proposal_id, source_peer); + retain_active_proposal_sources(state, book); + Ok(outcome) +} + +fn retain_active_proposal_sources(state: &mut DriveState, book: &ReassemblyBook) { + let active: HashSet<[u8; 32]> = book.proposal_ids().collect(); + state.proposal_source.retain(|proposal_id, _| active.contains(proposal_id)); +} + +fn expire_reassembly(state: &mut DriveState, book: &mut ReassemblyBook, now: Instant) { + let _ = book.expire(now); + retain_active_proposal_sources(state, book); +} + +fn reply_queue_at_capacity(len: usize) -> bool { + len >= MAX_CONCURRENT_DIRECT_EXCHANGES +} + +fn has_validator_leaf(state: &DriveState, node_id: &NodeId) -> bool { + state.validator_leaves.contains_key(node_id) +} + +fn replace_validator_leaves( + state: &mut DriveState, + leaves: Vec, +) -> Result<(), NetworkError> { + if leaves.len() > MAX_VALIDATORS_PER_REALM { + return Err(NetworkError::Rejected(format!( + "validator leaf count {} exceeds {MAX_VALIDATORS_PER_REALM}", + leaves.len() + ))); + } + state.validator_leaves.clear(); + for leaf in leaves { + state.validator_leaves.insert(leaf.node_id, leaf); + } + Ok(()) +} + fn vote_is_authenticated(state: &DriveState, vote: &Vote) -> bool { let Some(auth) = state.vote_auth.get(&vote.proposal_id) else { return false; @@ -894,7 +1264,7 @@ mod tests { assert!(!feed_vote_waiters(&mut state, &garbage)); assert!(state.vote_backlog.is_empty()); - register_proposal_vote_auth(&mut state, &proposal, leaf_bls_keys); + register_proposal_vote_auth(&mut state, &proposal, leaf_bls_keys, Instant::now()); let (response, mut result) = oneshot::channel(); state.vote_waiters.push(VoteWaiter { proposal_id: proposal.proposal_id, @@ -927,6 +1297,7 @@ mod tests { (proposal.proposer_sub_id, proposer.public_key()), (2, remote.public_key()), ], + Instant::now(), ); let canonical = vote_message( proposal.chain_id, @@ -967,7 +1338,12 @@ mod tests { let proposal = test_proposal([0x33; 32]); let secret = bls_secret(4); let remote_sub_id = 2; - register_proposal_vote_auth(&mut state, &proposal, vec![(remote_sub_id, secret.public_key())]); + register_proposal_vote_auth( + &mut state, + &proposal, + vec![(remote_sub_id, secret.public_key())], + Instant::now(), + ); let canonical = vote_message( proposal.chain_id, proposal.realm_id, @@ -991,4 +1367,73 @@ mod tests { assert!(state.vote_auth.get(&proposal.proposal_id).is_none()); assert!(state.vote_backlog.get(&proposal.proposal_id).is_none()); } + + #[test] + fn unfinished_proposals_keep_reassembly_and_source_maps_bounded() { + let mut book = ReassemblyBook::new( + MAX_IN_FLIGHT_PROPOSALS, + MAX_PROPOSAL_CHUNK_BYTES, + Duration::from_secs(1_800), + ); + let mut state = DriveState::new(); + let peer = Keypair::generate_ed25519().public().to_peer_id(); + let now = Instant::now(); + for index in 0..32u8 { + let mut proposal_id = [0u8; 32]; + proposal_id[0] = index; + admit_proposal_start( + &mut state, + &mut book, + test_proposal(proposal_id), + 1, + 1, + peer, + now, + ) + .expect("valid start"); + } + assert_eq!(book.active_count(), MAX_IN_FLIGHT_PROPOSALS); + assert_eq!(state.proposal_source.len(), book.active_count()); + assert!(state.proposal_source.len() <= MAX_IN_FLIGHT_PROPOSALS); + + expire_reassembly(&mut state, &mut book, now + Duration::from_secs(1_801)); + assert_eq!(book.active_count(), 0); + assert_eq!(state.proposal_source.len(), 0); + } + + #[test] + fn vote_auth_without_waiter_is_capped_and_expires() { + let mut state = DriveState::new(); + let now = Instant::now(); + for index in 0..16u8 { + let mut proposal_id = [0u8; 32]; + proposal_id[0] = index; + register_proposal_vote_auth(&mut state, &test_proposal(proposal_id), Vec::new(), now); + } + assert!(state.vote_auth.len() <= MAX_VOTE_AUTH); + assert!(state.vote_backlog.len() <= MAX_VOTE_AUTH); + expire_idle_vote_auth(&mut state, now + Duration::from_secs(VOTE_AUTH_TTL_SECS)); + assert!(state.vote_auth.is_empty()); + assert!(state.vote_backlog.is_empty()); + } + + #[test] + fn reply_queue_capacity_matches_direct_exchange_limit() { + assert!(!reply_queue_at_capacity(0)); + assert!(!reply_queue_at_capacity(MAX_CONCURRENT_DIRECT_EXCHANGES - 1)); + assert!(reply_queue_at_capacity(MAX_CONCURRENT_DIRECT_EXCHANGES)); + assert!(reply_queue_at_capacity(MAX_CONCURRENT_DIRECT_EXCHANGES + 1)); + } + + #[test] + fn missing_validator_leaf_closes_as_not_a_validator() { + let mut state = DriveState::new(); + let node = NodeId::from_keypair(&Keypair::generate_ed25519()).expect("ed25519"); + assert!(!has_validator_leaf(&state, &node)); + let leaf = ValidatorLeaf::new(1, node, bls_secret(1).public_key()); + replace_validator_leaves(&mut state, vec![leaf]).expect("leaf count fits"); + assert!(has_validator_leaf(&state, &node)); + let other = NodeId::from_keypair(&Keypair::generate_ed25519()).expect("ed25519"); + assert!(!has_validator_leaf(&state, &other)); + } } diff --git a/psy_node_common/src/realm/network/mod.rs b/psy_node_common/src/realm/network/mod.rs index 6d4d578e..becb70d6 100644 --- a/psy_node_common/src/realm/network/mod.rs +++ b/psy_node_common/src/realm/network/mod.rs @@ -25,10 +25,13 @@ mod startup; pub use behaviour::{ - add_bootnode_address, add_known_address, proposal_topic, vote_topic, RealmBehaviour, + add_bootnode_address, proposal_topic, register_explicit_peer, vote_topic, RealmBehaviour, IDENTIFY_PROTOCOL_ID, }; -pub use codec::{DirectBodyCodec, EndCapForwardCodec, EndCapForwardRequest, DIRECT_BODY_PROTOCOL_ID, END_CAP_FORWARD_PROTOCOL_ID}; +pub use codec::{ + BodyChunkCodec, EndCapForwardCodec, EndCapForwardRequest, ProposalLookupCodec, + DIRECT_BODY_PROTOCOL_ID, END_CAP_FORWARD_PROTOCOL_ID, PROPOSAL_LOOKUP_PROTOCOL_ID, +}; pub use config::{ generate_bls_secret_file, generate_ed25519_identity_file, load_bls_secret_key, load_ed25519_identity_key, RealmNetworkConfig, BOOTNODE_MIN_CIRCUIT_BYTES, @@ -45,9 +48,10 @@ pub use startup::{build_optional_realm_network, parse_bootnode, OptionalRealmNet use libp2p::request_response; use libp2p::{identity, noise, tcp, yamux, Swarm, SwarmBuilder}; use psy_data::p2p::{ - BlsPublicKey, DirectBodyResponse, EndCapForwardHeader, EndCapForwardResponse, NodeId, - Proposal, Vote, + BlsPublicKey, BodyChunkRequest, BodyChunkResponse, EndCapForwardHeader, EndCapForwardResponse, + NodeId, Proposal, ProposalLookupRequest, ProposalLookupResponse, ValidatorLeaf, Vote, }; +use std::collections::HashMap; use std::time::Duration; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; @@ -59,8 +63,8 @@ pub enum NetworkError { Configuration(String), #[error("network key file {path}: {details}")] KeyFile { path: String, details: String }, - #[error("unauthorized NodeId {0}")] - Unauthorized(NodeId), + #[error("NodeId {0} has no validator leaf under the checkpoint validator tree")] + NotAValidator(NodeId), #[error("protocol error: {0}")] Protocol(#[from] psy_data::p2p::ProtocolError), #[error("libp2p behaviour error: {0}")] @@ -106,11 +110,17 @@ pub enum RealmNetworkCommand { vote: Vote, response: oneshot::Sender>, }, - /// Respond to an inbound direct-body range request. - ServeBody { - request_id: request_response::InboundRequestId, - response_body: DirectBodyResponse, - response: oneshot::Sender>, + /// Ask one peer for candidates across a window of root pairs. + LookupProposal { + destination: NodeId, + request: ProposalLookupRequest, + response: oneshot::Sender>, + }, + /// Ask one peer for one direct-body range. + RequestBody { + destination: NodeId, + request: BodyChunkRequest, + response: oneshot::Sender>, }, /// Block until `threshold` distinct votes for `proposal_id` arrive, or /// `timeout` elapses. Fail-closed: timeout or a closed waiter is an error. @@ -120,6 +130,10 @@ pub enum RealmNetworkCommand { timeout: Duration, response: oneshot::Sender, NetworkError>>, }, + SetValidatorLeaves { + leaves: Vec, + response: oneshot::Sender>, + }, } /// Network → application events. The driving loop emits these for the @@ -128,7 +142,7 @@ pub enum RealmNetworkCommand { #[derive(Debug)] pub enum RealmNetworkEvent { /// An EndCap forward request was received. The application validates the - /// header/payload and replies with `EndCapForwardResponse` on `reply`. + /// header, input, and proof, and replies with `EndCapForwardResponse` on `reply`. EndCapReceived { request_id: request_response::InboundRequestId, source: NodeId, @@ -143,6 +157,18 @@ pub enum RealmNetworkEvent { proposal: Proposal, body: VerifiedProposalBody, }, + /// Inbound ProposalLookup from a current validator leaf. Drop `reply` to close the stream. + LookupReceived { + source: NodeId, + request: ProposalLookupRequest, + reply: oneshot::Sender, + }, + /// Inbound direct-body range from a current validator leaf. Drop `reply` to close the stream. + DirectBodyReceived { + source: NodeId, + request: BodyChunkRequest, + reply: oneshot::Sender, + }, /// A vote was received on the Realm vote topic. VoteReceived { source: NodeId, @@ -167,6 +193,15 @@ impl std::fmt::Debug for RealmNetworkCommands { } impl RealmNetworkCommands { + /// Builds a command handle over an existing command channel. The caller owns + /// the matching receiver and must drive it. + pub fn from_channel(commands: mpsc::Sender, local_node_id: NodeId) -> Self { + Self { + commands, + local_node_id, + } + } + pub fn local_node_id(&self) -> NodeId { self.local_node_id } @@ -208,14 +243,27 @@ impl RealmNetworkCommands { .await? } - pub async fn serve_body( + pub async fn lookup_proposal( &self, - request_id: request_response::InboundRequestId, - response_body: DirectBodyResponse, - ) -> Result<(), NetworkError> { - self.request(|response| RealmNetworkCommand::ServeBody { - request_id, - response_body, + destination: NodeId, + request: ProposalLookupRequest, + ) -> Result { + self.request(|response| RealmNetworkCommand::LookupProposal { + destination, + request, + response, + }) + .await? + } + + pub async fn request_body( + &self, + destination: NodeId, + request: BodyChunkRequest, + ) -> Result { + self.request(|response| RealmNetworkCommand::RequestBody { + destination, + request, response, }) .await? @@ -247,6 +295,14 @@ impl RealmNetworkCommands { .map_err(|_| NetworkError::CommandChannelClosed)?; rx.await.map_err(|_| NetworkError::ResponseChannelClosed) } + + pub async fn set_validator_leaves( + &self, + leaves: Vec, + ) -> Result<(), NetworkError> { + self.request(|response| RealmNetworkCommand::SetValidatorLeaves { leaves, response }) + .await? + } } /// Application-facing handle. Commands are cloneable; the event receiver is @@ -308,12 +364,20 @@ impl RealmNetworkHandle { self.commands.publish_vote(vote).await } - pub async fn serve_body( + pub async fn lookup_proposal( &self, - request_id: request_response::InboundRequestId, - response_body: DirectBodyResponse, - ) -> Result<(), NetworkError> { - self.commands.serve_body(request_id, response_body).await + destination: NodeId, + request: ProposalLookupRequest, + ) -> Result { + self.commands.lookup_proposal(destination, request).await + } + + pub async fn request_body( + &self, + destination: NodeId, + request: BodyChunkRequest, + ) -> Result { + self.commands.request_body(destination, request).await } pub async fn wait_votes( @@ -339,6 +403,7 @@ pub struct RealmNetwork { command_rx: mpsc::Receiver, event_tx: mpsc::Sender, reassembly: ReassemblyBook, + validator_leaves: HashMap, } impl std::fmt::Debug for RealmNetwork { @@ -390,10 +455,7 @@ impl RealmNetwork { ); let handle = RealmNetworkHandle { - commands: RealmNetworkCommands { - commands: command_tx, - local_node_id, - }, + commands: RealmNetworkCommands::from_channel(command_tx, local_node_id), events: event_rx, }; @@ -406,11 +468,19 @@ impl RealmNetwork { command_rx, event_tx, reassembly, + validator_leaves: HashMap::new(), }, handle, )) } + pub fn set_validator_leaves(&mut self, leaves: Vec) { + self.validator_leaves = leaves + .into_iter() + .map(|leaf| (leaf.node_id, leaf)) + .collect(); + } + /// Build a Realm network, loading the Ed25519 identity key from /// `config.identity_key_path`. pub fn build( diff --git a/psy_node_common/src/realm/network/reassembly.rs b/psy_node_common/src/realm/network/reassembly.rs index e493f17a..f2db8d63 100644 --- a/psy_node_common/src/realm/network/reassembly.rs +++ b/psy_node_common/src/realm/network/reassembly.rs @@ -54,18 +54,7 @@ impl VerifiedProposalBody { &self.storage.bytes } - pub fn read_range(&self, offset: u64, length: usize) -> io::Result> { - let end = offset - .checked_add(length as u64) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "proposal body range overflow"))?; - if end > self.len() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "proposal body range out of bounds")); - } - Ok(self.storage.bytes[offset as usize..end as usize].to_vec()) - } - - /// Durably persist the verified body to `destination` (blocking std::fs). - pub fn persist_to(&self, destination: &Path) -> io::Result<()> { + pub fn save_to(&self, destination: &Path) -> io::Result<()> { if let Some(parent) = destination.parent() { std::fs::create_dir_all(parent)?; } @@ -296,7 +285,7 @@ impl ReassemblyBook { } /// Begin a new reassembly. Returns `Duplicate` if one already exists for - /// the proposal, or `Inserted` after evicting the oldest entry when the + /// the proposal, or `Inserted` after evicting the oldest reassembly when the /// in-flight bound is reached. pub fn start( &mut self, diff --git a/psy_node_common/src/realm/network/startup.rs b/psy_node_common/src/realm/network/startup.rs index db78b639..e2dd1bc5 100644 --- a/psy_node_common/src/realm/network/startup.rs +++ b/psy_node_common/src/realm/network/startup.rs @@ -10,7 +10,9 @@ use crate::realm::network::{ use libp2p::multiaddr::Protocol; use libp2p::{Multiaddr, PeerId}; use parth_common::realm_rotation::RealmRotationConfig; -use psy_data::p2p::{BlsSecretKey, NodeId}; +use psy_data::p2p::{ + BlsSecretKey, NodeId, MAX_IN_FLIGHT_PROPOSALS, MAX_PROPOSAL_CHUNK_BYTES, REASSEMBLY_EXPIRY_SECS, +}; use std::str::FromStr; /// Parsed optional Realm network plus the rotation/BLS material the diff --git a/psy_node_common/src/realm/processor/catchup.rs b/psy_node_common/src/realm/processor/catchup.rs new file mode 100644 index 00000000..4b8d9aca --- /dev/null +++ b/psy_node_common/src/realm/processor/catchup.rs @@ -0,0 +1,388 @@ +//! Catch-up window fetching: peer selection, staged windows, and body downloads. + +use std::collections::HashMap; + +use anyhow::Context; +use parth_core::protocol::core_types::Q256BitHash; +use psy_data::p2p::{ + sha256, BodyChunkRequest, BodyChunkResponse, NodeId, Proposal, ProposalLookupEntry, + ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, + BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, PROPOSAL_LOOKUP_CONCURRENCY, + PROPOSAL_LOOKUP_ROUND_SECS, PROPOSAL_LOOKUP_TIMEOUT_SECS, PROPOSAL_LOOKUP_WINDOW_PAIRS, +}; + +use crate::realm::network::RealmNetworkCommands; +use crate::realm::processor::proposal_store::{ProposalStore, StagedProposal}; + +pub const CATCHUP_PAIR_ATTEMPTS: usize = 3; + +/// One window transition after the fetch stage: staged bytes awaiting verification. +pub enum TransitionFetchOutcome { + Staged(RealmTransition, StagedProposal), + Absent(RealmTransition), + Failed(RealmTransition, anyhow::Error), +} + +/// Next coordinator-authenticated realm transition after `last_committed`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum UnappliedTransition { + None { accounted_checkpoint: u64 }, + Real { pair: RealmTransition, included_checkpoint: u64 }, +} + +/// Last-modified events must be chronological. Equal roots are leaf rewrites, not bodies. +/// A→B→A keeps the A→B hop because that first value differs. +pub(crate) fn first_unapplied_transition( + last_committed: u64, + last_committed_root: [u8; 32], + last_modifieds: &[(u64, [u8; 32])], +) -> UnappliedTransition { + let mut old_root = last_committed_root; + let mut accounted_checkpoint = last_committed; + for &(checkpoint_id, new_root) in last_modifieds { + if checkpoint_id <= last_committed { + continue; + } + if new_root != old_root { + return UnappliedTransition::Real { + pair: RealmTransition { old_root, new_root }, + included_checkpoint: checkpoint_id, + }; + } + accounted_checkpoint = checkpoint_id; + old_root = new_root; + } + UnappliedTransition::None { accounted_checkpoint } +} + +/// Peer set chosen once per catch-up batch: one primary and at most one backup. +/// +/// The lookup key is the proposal's (old_root, new_root) pair, so any peer that +/// stored the proposal can answer; peers are tried in ascending validator sub-id +/// order and never rescanned per item. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatchupPeers { + primary: NodeId, + backup: Option, +} + +impl CatchupPeers { + pub fn select(members: &[(u16, NodeId)], local_sub_id: u16) -> anyhow::Result { + let mut unique_members: Vec<(u16, NodeId)> = members + .iter() + .copied() + .filter(|(sub_id, _)| *sub_id != local_sub_id) + .collect(); + unique_members.sort_by_key(|(sub_id, _)| *sub_id); + unique_members.dedup_by_key(|(sub_id, _)| *sub_id); + let mut iter = unique_members.into_iter(); + let primary = iter + .next() + .ok_or_else(|| anyhow::anyhow!("no other validator peer at this checkpoint"))? + .1; + Ok(Self { + primary, + backup: iter.next().map(|(_, peer)| peer), + }) + } + +} +/// Stage one proposal per pair: the primary peer answers the whole window, the +/// backup answers only when the primary fails, and never more than one backup. +pub async fn stage_transition_blocks( + client: &RealmNetworkCommands, + store: &ProposalStore, + peers: &CatchupPeers, + chain_id: u64, + realm_id: u32, + needed: &[RealmTransition], + rejected_proposal_ids: &[[u8; 32]], +) -> Vec { + let mut staged = Vec::with_capacity(needed.len()); + let mut windows = needed.chunks(PROPOSAL_LOOKUP_WINDOW_PAIRS); + let mut tasks = futures::stream::FuturesUnordered::new(); + while tasks.len() < PROPOSAL_LOOKUP_CONCURRENCY { + let Some(window) = windows.next() else { + break; + }; + tasks.push(stage_transition_window( + client, + store, + peers, + chain_id, + realm_id, + window, + rejected_proposal_ids, + )); + } + while let Some(window_staged) = futures::StreamExt::next(&mut tasks).await { + staged.extend(window_staged); + if let Some(window) = windows.next() { + tasks.push(stage_transition_window( + client, + store, + peers, + chain_id, + realm_id, + window, + rejected_proposal_ids, + )); + } + } + staged +} + +async fn stage_transition_window( + client: &RealmNetworkCommands, + store: &ProposalStore, + peers: &CatchupPeers, + chain_id: u64, + realm_id: u32, + window: &[RealmTransition], + rejected_proposal_ids: &[[u8; 32]], +) -> Vec { + let deadline = tokio::time::Instant::now() + + std::time::Duration::from_secs(PROPOSAL_LOOKUP_TIMEOUT_SECS); + let (peer, response) = match lookup_pending_transitions(client, peers, chain_id, realm_id, window, deadline).await + { + Ok(answered) => answered, + Err(error) => { + let message = format!("{error:#}"); + return window + .iter() + .map(|transition| TransitionFetchOutcome::Failed(*transition, anyhow::anyhow!("{message}"))) + .collect(); + } + }; + if response.status == ProposalLookupStatus::Truncated { + if window.len() <= 1 { + return window + .iter() + .map(|transition| { + TransitionFetchOutcome::Failed( + *transition, + anyhow::anyhow!("peer truncated a single-transition window"), + ) + }) + .collect(); + } + let (head, tail) = window.split_at(window.len() / 2); + let mut staged = Box::pin(stage_transition_window( + client, + store, + peers, + chain_id, + realm_id, + head, + rejected_proposal_ids, + )) + .await; + staged.extend( + Box::pin(stage_transition_window( + client, + store, + peers, + chain_id, + realm_id, + tail, + rejected_proposal_ids, + )) + .await, + ); + return staged; + } + let mut answers: Vec = response.entries; + let mut staged = Vec::with_capacity(window.len()); + for transition in window { + let candidates = answers + .iter() + .position(|entry| entry.transition == *transition) + .map(|index| answers.swap_remove(index).candidates) + .unwrap_or_default(); + let Some(candidate) = candidates + .into_iter() + .find(|candidate| !rejected_proposal_ids.contains(&candidate.proposal_id)) + else { + staged.push(TransitionFetchOutcome::Absent(*transition)); + continue; + }; + match download_proposal_body(client, peer, &candidate, deadline).await { + Ok(body) => match store.create_staged(&candidate, &body).await { + Ok(staged_proposal) => staged.push(TransitionFetchOutcome::Staged(*transition, staged_proposal)), + Err(error) => staged.push(TransitionFetchOutcome::Failed(*transition, error)), + }, + Err(error) => staged.push(TransitionFetchOutcome::Failed(*transition, error)), + } + } + staged +} + +async fn lookup_pending_transitions( + client: &RealmNetworkCommands, + peers: &CatchupPeers, + chain_id: u64, + realm_id: u32, + window: &[RealmTransition], + deadline: tokio::time::Instant, +) -> anyhow::Result<(NodeId, ProposalLookupResponse)> { + let request = ProposalLookupRequest { + chain_id, + realm_id, + pairs: window.to_vec(), + }; + let mut last_error = None; + for peer in [Some(peers.primary), peers.backup].into_iter().flatten() { + match lookup_pending_transitions_from_peer(client, peer, &request, deadline).await { + Ok(response) => return Ok((peer, response)), + Err(error) => { + tracing::warn!( + "catch-up window lookup peer={peer} pairs={} error={error:#}", + window.len() + ); + last_error = Some(error); + } + } + } + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("no catch-up peer available"))) +} + +async fn lookup_pending_transitions_from_peer( + client: &RealmNetworkCommands, + peer: NodeId, + request: &ProposalLookupRequest, + deadline: tokio::time::Instant, +) -> anyhow::Result { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + anyhow::ensure!(!remaining.is_zero(), "catch-up lookup timeout"); + let round = remaining.min(std::time::Duration::from_secs(PROPOSAL_LOOKUP_ROUND_SECS)); + match tokio::time::timeout(round, client.lookup_proposal(peer, request.clone())).await { + Ok(Ok(response)) => Ok(response), + Ok(Err(error)) => anyhow::bail!("{error}"), + Err(_) => anyhow::bail!("catch-up lookup timeout"), + } +} + +async fn download_proposal_body( + client: &RealmNetworkCommands, + peer: NodeId, + proposal: &Proposal, + deadline: tokio::time::Instant, +) -> anyhow::Result> { + let mut offset = 0u64; + let mut body = Vec::new(); + let mut expected_hash = None; + let mut expected_len = None; + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + anyhow::ensure!(!remaining.is_zero(), "direct range timeout"); + let request = BodyChunkRequest { + proposal_id: proposal.proposal_id, + offset, + max_bytes: BODY_CHUNK_MAX_BYTES, + }; + let round = remaining.min(std::time::Duration::from_secs(PROPOSAL_LOOKUP_ROUND_SECS)); + let response = match tokio::time::timeout(round, client.request_body(peer, request)).await { + Ok(Ok(response)) => response, + Ok(Err(error)) => anyhow::bail!("{error}"), + Err(_) => anyhow::bail!("direct range timeout"), + }; + if let Some(body_hash) = expected_hash { + anyhow::ensure!(response.body_hash == body_hash, "direct range body_hash changed"); + } else { + expected_hash = Some(response.body_hash); + } + if let Some(body_len) = expected_len { + anyhow::ensure!(response.body_len == body_len, "direct range body_len changed"); + } else { + anyhow::ensure!( + response.body_len <= MAX_PROPOSAL_BODY_BYTES as u64, + "direct range body_len exceeds maximum" + ); + expected_len = Some(response.body_len); + } + anyhow::ensure!(response.offset == offset, "direct range offset mismatch"); + anyhow::ensure!( + !response.data.is_empty() || response.eof, + "direct range empty non-final chunk" + ); + let next = offset + .checked_add(response.data.len() as u64) + .ok_or_else(|| anyhow::anyhow!("direct range offset overflow"))?; + anyhow::ensure!(next <= response.body_len, "direct range past body_len"); + anyhow::ensure!( + response.eof == (next == response.body_len), + "direct range eof mismatch" + ); + body.extend_from_slice(&response.data); + offset = next; + if response.eof { + anyhow::ensure!(sha256(&body) == proposal.body_hash, "direct range body hash mismatch"); + anyhow::ensure!( + response.body_hash == proposal.body_hash, + "direct range declared hash mismatch" + ); + return Ok(body); + } + } +} + +#[cfg(test)] +mod tests { + use super::{first_unapplied_transition, UnappliedTransition}; + use psy_data::p2p::RealmTransition; + + const ROOT_A: [u8; 32] = [0xA; 32]; + const ROOT_B: [u8; 32] = [0xB; 32]; + + #[test] + fn first_unapplied_transition_skips_identity_rewrite() { + let outcome = first_unapplied_transition(15, ROOT_A, &[(16, ROOT_A)]); + assert_eq!( + outcome, + UnappliedTransition::None { + accounted_checkpoint: 16 + } + ); + } + + #[test] + fn first_unapplied_transition_keeps_cycle_first_hop() { + let outcome = first_unapplied_transition(15, ROOT_A, &[(20, ROOT_B), (50, ROOT_A)]); + assert_eq!( + outcome, + UnappliedTransition::Real { + pair: RealmTransition { + old_root: ROOT_A, + new_root: ROOT_B, + }, + included_checkpoint: 20, + } + ); + } + + #[test] + fn first_unapplied_transition_skips_identity_then_takes_real() { + let outcome = first_unapplied_transition(15, ROOT_A, &[(16, ROOT_A), (50, ROOT_B)]); + assert_eq!( + outcome, + UnappliedTransition::Real { + pair: RealmTransition { + old_root: ROOT_A, + new_root: ROOT_B, + }, + included_checkpoint: 50, + } + ); + } + + #[test] + fn first_unapplied_transition_empty_is_accounted_at_committed() { + let outcome = first_unapplied_transition(15, ROOT_A, &[]); + assert_eq!( + outcome, + UnappliedTransition::None { + accounted_checkpoint: 15 + } + ); + } +} diff --git a/psy_node_common/src/realm/processor/consensus.rs b/psy_node_common/src/realm/processor/consensus.rs index 8dc3aff1..0a972340 100644 --- a/psy_node_common/src/realm/processor/consensus.rs +++ b/psy_node_common/src/realm/processor/consensus.rs @@ -33,7 +33,7 @@ use psy_data::{ p2p::{ aggregate_signatures, bitmap_get, bitmap_set, sha256, vote_message, BlsPublicKey, BlsSecretKey, BlsSignature, Certificate, ProtocolError, ProtocolResult, Proposal, Vote, - MAX_INCLUSION_LAG_CHECKPOINTS, MAX_VALIDATORS_PER_REALM, MIN_VALIDATORS_PER_REALM, + MAX_VALIDATORS_PER_REALM, MIN_VALIDATORS_PER_REALM, replication_threshold, }, prepared_block::realm::PsyPreparedRealmBlockStateUpdates, @@ -89,8 +89,8 @@ pub fn decode_proposal_body( Ok(DecodedProposalBody { output, proof, state_updates, worker_tag }) } -/// Verify the in-band FFS roots against the decoded GUTA output. -pub fn verify_state_updates_match_guta_output( +/// Check declared roots only; recovery verifies every durable FFS record separately. +pub fn require_declared_roots_match_zk_output( state_updates: &PsyPreparedRealmBlockStateUpdates, output: &RealmFinalizeGUTAPublicOutput, ) -> anyhow::Result<()> @@ -185,7 +185,7 @@ where "Realm finalize output fee user is not the scheduled proposer's validator leaf user" ); let state_updates = decode_proposal_state_updates::(&decoded.state_updates)?; - verify_state_updates_match_guta_output(&state_updates, &output)?; + require_declared_roots_match_zk_output(&state_updates, &output)?; let worker_tag = N::QHash::from_owned_32bytes(decoded.worker_tag); let expected_public_inputs_hash = finalize_public_input_hash::(&output, &worker_tag); @@ -369,37 +369,21 @@ pub fn validator_tree_root_matches_proof_base( proposal_root == proof_base_root } -/// Inclusion lag of a GUTA Proposal proof-base checkpoint against the -/// coordinator's inclusion checkpoint: the proof base must strictly precede -/// inclusion (`lag >= 1`) and must not be older than -/// [`MAX_INCLUSION_LAG_CHECKPOINTS`]. Returns `Some(lag)` when admissible. -pub fn inclusion_lag_within_limit( - base_checkpoint_id: u64, - inclusion_checkpoint_id: u64, -) -> Option { - inclusion_checkpoint_id - .checked_sub(base_checkpoint_id) - .filter(|lag| (1..=MAX_INCLUSION_LAG_CHECKPOINTS).contains(lag)) -} - /// True when the certificate's signer set includes the proposal's proposer. pub fn certificate_includes_proposer(certificate: &Certificate, proposer_sub_id: u16) -> bool { certificate.signer_sub_ids().contains(&proposer_sub_id) } -/// True when the collected votes satisfy the proposal's replication wait: -/// at least `replication_threshold(n)` distinct signers and — for realms with -/// two or more validators — at least one signer other than the proposer (a -/// proposal must not be certified by the proposer alone). -pub fn votes_meet_wait(n: usize, proposer_sub_id: u16, signer_sub_ids: &[u16]) -> bool { +/// True when collected votes meet `replication_threshold(n)` distinct signers. +/// The proposer's own vote counts toward `ceil(n / 2)`. For n == 2 that +/// threshold is 1, so the proposer may certify alone. For n >= 3 the +/// threshold is at least 2, which already requires a non-proposer signer. +/// A separate "must include a non-proposer" rule would bind only two-validator +/// realms and stall them after a single-node fault. Production realms have +/// more than two validators, so that extra rule is not restored. +pub fn votes_meet_wait(n: usize, signer_sub_ids: &[u16]) -> bool { let unique: HashSet = signer_sub_ids.iter().copied().collect(); - if unique.len() < replication_threshold(n) { - return false; - } - if n >= 2 && !unique.iter().any(|sub_id| *sub_id != proposer_sub_id) { - return false; - } - true + unique.len() >= replication_threshold(n) } #[cfg(test)] @@ -463,6 +447,45 @@ mod tests { + #[test] + fn proposal_identity_mutations_require_rehashing_but_transport_is_not_proof_validation() { + let (output, proof, backup) = sample_body_sections(); + let (proposal, body) = + proposal_with_body(TEST_CHAIN_ID, 3, 99, 1, [1u8; 32], &output, &proof, &backup); + // These are transport-only fixtures, not valid circuit proofs or state updates. + for offset in [32usize, 48, 114] { + let mut changed_output = output.clone(); + changed_output[offset] = 1; + let (rehashed, changed_body) = proposal_with_body( + TEST_CHAIN_ID, 3, 99, 1, [1u8; 32], &changed_output, &proof, &backup, + ); + assert!(decode_proposal_body(&proposal, &changed_body).is_err()); + let mut body_hash_only = proposal.clone(); + body_hash_only.body_hash = rehashed.body_hash; + assert!(decode_proposal_body(&body_hash_only, &changed_body).is_err()); + assert_ne!(body_hash_only.compute_proposal_id(), proposal.proposal_id); + assert_eq!(rehashed.compute_proposal_id(), rehashed.proposal_id); + assert_ne!(rehashed.proposal_id, proposal.proposal_id); + let decoded = decode_proposal_body(&rehashed, &changed_body).unwrap(); + assert_eq!(decoded.proof, proof); + assert_eq!(decoded.state_updates, backup); + assert_eq!(&decoded.output[218..], &output[218..]); + let mut deleted_body = body.clone(); + deleted_body.drain(4 + offset..4 + offset + if offset == 48 { 2 } else { 8 }); + let mut deleted_proposal = proposal.clone(); + deleted_proposal.body_hash = sha256(&deleted_body); + deleted_proposal.proposal_id = deleted_proposal.compute_proposal_id(); + assert!(decode_proposal_body(&deleted_proposal, &deleted_body).is_err()); + } + for proposer_sub_id in [0, 2] { + let mut changed = proposal.clone(); + changed.proposer_sub_id = proposer_sub_id; + assert_ne!(changed.compute_proposal_id(), proposal.proposal_id); + changed.proposal_id = changed.compute_proposal_id(); + assert!(decode_proposal_body(&changed, &body).is_ok()); + } + } + #[test] fn decode_proposal_body_roundtrips_canonical_body() { let (output, proof, state_updates) = sample_body_sections(); @@ -727,26 +750,6 @@ mod tests { assert!(!validator_tree_root_matches_proof_base(&[0u8; 32], &[7u8; 32])); } - #[test] - fn inclusion_lag_within_limit_accepts_one_to_max() { - assert_eq!(inclusion_lag_within_limit(10, 11), Some(1)); - assert_eq!(inclusion_lag_within_limit(10, 26), Some(16)); - } - - #[test] - fn inclusion_lag_within_limit_rejects_zero_lag() { - // The proof base must strictly precede the inclusion checkpoint. - assert_eq!(inclusion_lag_within_limit(10, 10), None); - } - - #[test] - fn inclusion_lag_within_limit_rejects_lag_beyond_max_and_underflow() { - // lag 17 > MAX_INCLUSION_LAG_CHECKPOINTS (16). - assert_eq!(inclusion_lag_within_limit(10, 27), None); - // A proof base after the inclusion checkpoint cannot compute a lag. - assert_eq!(inclusion_lag_within_limit(20, 10), None); - } - #[test] fn certificate_includes_proposer_requires_proposer_signature() { let validator_sub_ids: [u16; 3] = [1, 2, 3]; @@ -768,34 +771,67 @@ mod tests { } #[test] - fn votes_meet_wait_requires_non_proposer_signer_when_n_is_two() { - // n = 2: the replication threshold is 1, but the proposer alone must - // not certify its own proposal. - assert!(!votes_meet_wait(2, 1, &[1])); - assert!(!votes_meet_wait(2, 1, &[1, 1])); - assert!(votes_meet_wait(2, 1, &[1, 2])); + fn votes_meet_wait_proposer_only_meets_threshold_when_n_is_two() { + assert!(votes_meet_wait(2, &[1])); + assert!(votes_meet_wait(2, &[1, 1])); + assert!(votes_meet_wait(2, &[1, 2])); } #[test] fn votes_meet_wait_proposer_only_is_enough_when_n_is_one() { - assert!(votes_meet_wait(1, 1, &[1])); + assert!(votes_meet_wait(1, &[1])); } #[test] fn votes_meet_wait_enforces_replication_threshold() { - // n = 3: ceil(3/2) = 2 distinct signers required. - assert!(!votes_meet_wait(3, 1, &[1])); - assert!(!votes_meet_wait(3, 1, &[1, 1])); // duplicates do not count twice - assert!(votes_meet_wait(3, 1, &[1, 2])); - assert!(votes_meet_wait(3, 1, &[2, 3])); + assert!(!votes_meet_wait(3, &[1])); + assert!(!votes_meet_wait(3, &[1, 1])); + assert!(votes_meet_wait(3, &[1, 2])); + assert!(votes_meet_wait(3, &[2, 3])); + } + + #[test] + fn certificate_rejects_tampered_proposal_id_despite_valid_signer_keys() { + // A certificate whose proposal_id field was replaced with a different + // proposal's id must be rejected: the identity comparison binds the + // certificate to the proposal before any signature check. + let validator_sub_ids: [u16; 3] = [1, 2, 3]; + let (secrets, keys) = build_validators(&validator_sub_ids); + let (output, proof, backup) = sample_body_sections(); + let (proposal, _body) = + proposal_with_body(TEST_CHAIN_ID, 3, 99, 1, [1u8; 32], &output, &proof, &backup); + let votes = signed_votes(&secrets, &validator_sub_ids, &proposal); + let mut cert = form_certificate(&proposal, &votes).expect("form"); + // Replace only the proposal_id with another valid proposal's id. + let (other, _) = + proposal_with_body(TEST_CHAIN_ID, 3, 99, 1, [2u8; 32], &output, &proof, &backup); + cert.proposal_id = other.proposal_id; + let err = validate_certificate(&proposal, &cert, &validator_sub_ids, &keys).unwrap_err(); + assert_eq!(err, ProtocolError::Message("certificate does not match proposal")); + } + + #[test] + fn certificate_rejects_bitmap_corruption_below_threshold() { + // Same signer aggregate, same proposal: clearing signer bits must + // reduce the replication count below ceil(n/2) and be rejected. + let validator_sub_ids: [u16; 3] = [1, 2, 3]; + let (secrets, keys) = build_validators(&validator_sub_ids); + let (output, proof, backup) = sample_body_sections(); + let (proposal, _body) = + proposal_with_body(TEST_CHAIN_ID, 3, 99, 1, [1u8; 32], &output, &proof, &backup); + let votes = signed_votes(&secrets, &validator_sub_ids, &proposal); + let mut cert = form_certificate(&proposal, &votes).expect("form"); + // Bits 1..=3 were set by form_certificate; clear bits 2 and 3 so only + // one signer remains, below ceil(3/2) = 2. + cert.signer_bitmap[0] &= !(0x08 | 0x04); + let err = validate_certificate(&proposal, &cert, &validator_sub_ids, &keys).unwrap_err(); + assert_eq!(err, ProtocolError::Message("certificate below replication threshold")); } #[test] fn votes_meet_wait_duplicate_signer_does_not_count_twice() { - // Three reported votes from two distinct signers still satisfy the - // n = 3 wait (threshold 2) with a non-proposer signer. - assert!(votes_meet_wait(3, 1, &[1, 1, 2])); - // n = 2 with only duplicated proposer votes has no non-proposer signer. - assert!(!votes_meet_wait(2, 1, &[1, 1])); + assert!(votes_meet_wait(3, &[1, 1, 2])); + assert!(votes_meet_wait(2, &[1, 1])); + assert!(!votes_meet_wait(3, &[1, 1, 1])); } } \ No newline at end of file diff --git a/psy_node_common/src/realm/processor/core/mod.rs b/psy_node_common/src/realm/processor/core/mod.rs index 58725277..e8b2c939 100644 --- a/psy_node_common/src/realm/processor/core/mod.rs +++ b/psy_node_common/src/realm/processor/core/mod.rs @@ -4,6 +4,8 @@ use psy_io::tokio::TokioLikeFileSystem; use psy_node_core::{ p2p::traits::realm_coordinantor::RealmCoordinatorClient, psy_core_db::traits::full::{PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore}, psy_temp_db::StandardProcessorTempDBStoreBase, queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, store::traits::proof_store::QParthProofStore }; +use std::sync::Arc; + use tokio::sync::mpsc; use crate::{ @@ -67,11 +69,11 @@ pub struct PsyRealmProcessor< /// Local validator BLS secret key used to sign the processor's own Vote. /// Required when P2P is enabled; `set_realm_p2p` wires it. pub bls_secret: Option, - pub proposal_rx: Option>, - /// Unresolved complete bodies remain owned here until commit and gatherer fast-forward succeed. - /// No eviction: memory use grows with distinct unresolved proposal IDs. - pub proposals: std::collections::BTreeMap<[u8; 32], crate::realm::network::ProposalWithBody>, - - - + pub proposal_store: std::sync::Arc, + pub baseline_replay_rx: Option< + mpsc::Receiver>, + >, + pub file_system: Arc, + pub guta_gatherer_backup_directory: String, + pub guta_gatherer_join: Option>>, } diff --git a/psy_node_common/src/realm/processor/core/process_block.rs b/psy_node_common/src/realm/processor/core/process_block.rs index f2000272..a5504805 100644 --- a/psy_node_common/src/realm/processor/core/process_block.rs +++ b/psy_node_common/src/realm/processor/core/process_block.rs @@ -15,7 +15,7 @@ use psy_data::{ header_extended::{GlobalUserTreeAggregatorHeaderWithTagValue, GlobalUserTreeAggregatorHeaderWithTagValueAndJobType}, realm_finalize::{ finalize_output_from_witness, finalize_reward_root63, realm_finalize_guta_chain_domain, - protocol_encode_finalize_output, RealmFinalizeBinding, RealmFinalizeGUTAInput, + protocol_encode_finalize_output, RealmFinalizeGUTAInput, }, }, node::node_proving_state::PsyNodeProvingState, @@ -29,7 +29,10 @@ use psy_data::{ use psy_io::tokio::TokioLikeFileSystem; use cf_utils::timer::TraceTimer; use psy_node_core::{ - p2p::{traits::realm_coordinantor::RealmCoordinatorClient, validator_lookup::load_realm_validators_from_tree}, + p2p::{ + traits::realm_coordinantor::RealmCoordinatorClient, + validator_lookup::{load_realm_validators_from_tree, validator_nodes_from_leaves}, + }, psy_core_db::traits::full::{PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore}, psy_temp_db::StandardProcessorTempDBStoreBase, queue::{ @@ -42,7 +45,10 @@ use psy_node_core::{ use crate::{ realm::{ processor::{ - consensus::{form_certificate, require_nonzero_validator_tree_root, sign_vote, validate_certificate, votes_meet_wait}, + consensus::{ + form_certificate, require_nonzero_validator_tree_root, sign_vote, validate_certificate, + votes_meet_wait, + }, core::PsyRealmProcessor, gatherers::realm_end_cap_gatherer::RealmGUTAEndCapGathererOutput, }, @@ -54,7 +60,7 @@ use crate::utils::persisted_artifact::wait_for_persisted_artifact; impl< N: QNetworkTypesConfig, - S: PsyRealmProcessorStore + Send + Sync, + S: PsyRealmProcessorStore + Send + Sync + 'static, STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter + PsyNodeCoreRewardsTagTreeStoreReader + Send + Sync, GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync + 'static, ProofWorkQueue: QStandardWorkerQueuePublisher + QStandardWorkerQueueSubscriber + Send + Sync + 'static, @@ -211,61 +217,44 @@ where Ok(guta_result) } - pub async fn sync_and_verify(&mut self) -> anyhow::Result<()> { - // Drain before any fallible sync/early return; never replace a distinct proposal. - if let Some(proposal_rx) = self.proposal_rx.as_mut() { - while let Ok(complete) = proposal_rx.try_recv() { - self.proposals.entry(complete.proposal.proposal_id).or_insert(complete); + pub async fn sync_and_verify(&mut self) -> anyhow::Result<()> + where + N::HasherBase: MerkleZeroHasher, + { + let mut replay_requests = Vec::new(); + if let Some(replay_rx) = self.baseline_replay_rx.as_mut() { + while let Ok(request) = replay_rx.try_recv() { + replay_requests.push(request); } } + for request in replay_requests { + let result = self + .db + .verify_state_updates_from_baseline(request.previous_checkpoint_id, &request.updates) + .await; + let _ = request.reply.send(result); + } self.db.sync_with_coordinator().await?; - // This also retries gatherer fast-forward after a successful durable commit. self.commit_included_proposal_ffs().await?; self.db.ensure_db_matches_coordinator_head().await } - async fn ensure_uncommitted_processing_ids(&mut self, checkpoint_id: u64) -> anyhow::Result<()> { - let pending_id = self.db.state.processing_unique_pending_id; - let mapped_checkpoint = self.db.db.get_checkpoint_id_for_unique_pending_id(pending_id).await?; - // commit_state writes this forward mapping first. Keep its IDs even if the - // reverse mapping or a later write failed; a retry must not allocate again. - if pending_id != 0 && mapped_checkpoint == Some(checkpoint_id) { - return Ok(()); - } - let (pending_id, proc_checkpoint_unique_id) = - if let Some(ids) = self.db.db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await? { - ids - } else if pending_id != 0 && mapped_checkpoint.is_none() { - return Ok(()); - } else { - self.db.db.inc_unique_pending_id(1).await? - }; - self.db.state.processing_unique_pending_id = pending_id; - self.db.state.processing_proc_checkpoint_unique_id = proc_checkpoint_unique_id; - self.db.temp_db.set_unique_pending_ids( - &self.db.state.realm_identifier, pending_id, proc_checkpoint_unique_id, - ).await?; - Ok(()) - } - - async fn commit_included_proposal_ffs(&mut self) -> anyhow::Result<()> { + async fn commit_included_proposal_ffs(&mut self) -> anyhow::Result<()> + where + N::HasherBase: MerkleZeroHasher, + { let latest_checkpoint_id = self.db.coordinator_client.rc_get_latest_checkpoint_id().await?; - let mapped_checkpoint = self.db.db.get_checkpoint_id_for_unique_pending_id( - self.db.state.processing_unique_pending_id, - ).await?; - // Resume a partially committed transition at its original checkpoint, even - // when the coordinator has already included a subsequent proposal. - let checkpoint_id = mapped_checkpoint - .filter(|id| *id > self.db.state.last_committed_checkpoint_id) - .unwrap_or(latest_checkpoint_id); + self.db.set_last_committed_realm_root_from_db().await?; let mut coordinator_realm_state = self.db.coordinator_client - .rc_get_realm_root_and_last_modified_checkpoint(checkpoint_id, self.db.state.realm_id_u64) + .rc_get_realm_root_and_last_modified_checkpoint(latest_checkpoint_id, self.db.state.realm_id_u64) .await?; - // Walk authenticated last-modified boundaries backwards to the first - // unapplied transition. Looking only at the tip can skip retained bodies - // (or hide a root that changed away and then returned to its old value). + let mut last_modifieds = Vec::new(); while coordinator_realm_state.checkpoint_id > self.db.state.last_committed_checkpoint_id { + last_modifieds.push(( + coordinator_realm_state.checkpoint_id, + coordinator_realm_state.value.into_owned_32bytes(), + )); let previous = self.db.coordinator_client .rc_get_realm_root_and_last_modified_checkpoint( coordinator_realm_state.checkpoint_id - 1, self.db.state.realm_id_u64, @@ -277,66 +266,149 @@ where "Coordinator realm last-modified checkpoint did not decrease during recovery"); coordinator_realm_state = previous; } - - let mut selected = None; - for (proposal_id, complete) in &self.proposals { - let decoded = crate::realm::processor::consensus::decode_proposal_body( - &complete.proposal, complete.body.as_bytes(), - ).map_err(|error| anyhow::anyhow!("Proposal {}: {error}", hex::encode(proposal_id)))?; - let updates = PsyPreparedRealmBlockStateUpdates::::psy_ser_from_slice(&decoded.state_updates)?; - // An already committed object takes precedence: its gatherer may still - // need FF, and a newer transition cannot safely run ahead of that FF. - if updates.new_realm_root == self.db.state.last_committed_realm_end_root - && updates.old_realm_root == self.db.state.last_committed_realm_start_root - { - coordinator_realm_state = self.db.coordinator_client - .rc_get_realm_root_and_last_modified_checkpoint( - self.db.state.last_committed_checkpoint_id, self.db.state.realm_id_u64, - ).await?; - anyhow::ensure!(coordinator_realm_state.value == updates.new_realm_root, - "Committed proposal {} no longer matches authenticated realm root", hex::encode(proposal_id)); - selected = Some((*proposal_id, updates, decoded.state_updates)); - break; + last_modifieds.reverse(); + let old_root = self.db.state.last_committed_realm_end_root.into_owned_32bytes(); + let (pair, included_checkpoint_id) = match crate::realm::processor::catchup::first_unapplied_transition( + self.db.state.last_committed_checkpoint_id, + old_root, + &last_modifieds, + ) { + crate::realm::processor::catchup::UnappliedTransition::None { accounted_checkpoint } => { + if accounted_checkpoint > self.db.state.last_committed_checkpoint_id { + self.db.state.last_committed_checkpoint_id = accounted_checkpoint; + self.db.shared_state.update_from_core_state(&self.db.state).await?; + } + return self.db.sync_to_coordinator_checkpoint_id(latest_checkpoint_id).await; } - if selected.is_none() && updates.new_realm_root == coordinator_realm_state.value - && updates.old_realm_root == self.db.state.last_committed_realm_end_root - { - selected = Some((*proposal_id, updates, decoded.state_updates)); + crate::realm::processor::catchup::UnappliedTransition::Real { pair, included_checkpoint } => { + (pair, included_checkpoint) + } + }; + coordinator_realm_state = self.db.coordinator_client + .rc_get_realm_root_and_last_modified_checkpoint(included_checkpoint_id, self.db.state.realm_id_u64) + .await?; + let coordinator_update = self.db.coordinator_client + .rc_get_realm_sync_info(included_checkpoint_id, self.db.state.realm_id_u64) + .await?; + let included = crate::realm::processor::recovery::CheckpointIdentity { + checkpoint_id: included_checkpoint_id, + checkpoint_hash: coordinator_update + .checkpoint_sync_info + .checkpoint_leaf_hash + .into_owned_32bytes(), + }; + let gathering_start = self.db.state.gathering_realm_start_root; + let mut selected = match self + .db + .verify_history_transition(&included, pair, None, &self.proposal_store) + .await + { + Ok(Some(verified)) if included.checkpoint_id > self.db.state.last_committed_checkpoint_id => { + Some( + self.db + .apply_history_proposal(&included, verified) + .await?, + ) + } + Ok(Some(verified)) => Some((verified.updates, verified.state_updates)), + Ok(None) => None, + Err(error) if crate::realm::processor::recovery::invalid_candidate_id(&error).is_some() => None, + Err(error) => return Err(error), + }; + if selected.is_none() { + if let Some(client) = self.p2p.as_ref() { + let (_, _, _, leaves) = self + .load_base_checkpoint_validators(self.db.state.last_committed_checkpoint_id) + .await?; + let validator_nodes = validator_nodes_from_leaves(&leaves); + let peers = crate::realm::processor::catchup::CatchupPeers::select( + &validator_nodes, + self.db.state.realm_sub_id_u64 as u16, + )?; + let staged = crate::realm::processor::catchup::stage_transition_blocks( + client, + &self.proposal_store, + &peers, + self.db.state.chain_id, + self.db.state.realm_id_u64 as u32, + &[pair], + &[], + ) + .await + .into_iter() + .find_map(|outcome| match outcome { + crate::realm::processor::catchup::TransitionFetchOutcome::Staged(_, staged) => Some(staged), + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => { + tracing::debug!( + "no peer offered pair=({},{})", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ); + None + } + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => { + tracing::warn!( + "peer fetch failed pair=({},{}) error={error:#}", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ); + None + } + }); + selected = match self + .db + .verify_history_transition(&included, pair, staged.as_ref(), &self.proposal_store) + .await + { + Ok(Some(verified)) => { + if let Some(staged) = staged { + self.proposal_store.install(staged).await?; + } + if included.checkpoint_id > self.db.state.last_committed_checkpoint_id { + Some( + self.db + .apply_history_proposal(&included, verified) + .await?, + ) + } else { + Some((verified.updates, verified.state_updates)) + } + } + Ok(None) => None, + Err(error) => { + if crate::realm::processor::recovery::invalid_candidate_id(&error).is_none() { + return Err(error); + } + None + } + }; } } - let Some((proposal_id, updates, updates_bytes)) = selected else { + let Some((updates, updates_bytes)) = selected else { anyhow::ensure!(coordinator_realm_state.value == self.db.state.last_committed_realm_end_root && coordinator_realm_state.checkpoint_id <= self.db.state.last_committed_checkpoint_id, - "Checkpoint {}: no complete proposal body for included realm transition {:?} -> {:?}; retaining {} unresolved proposals without advancing committed state", + "Checkpoint {}: no stored proposal for included realm transition {:?} -> {:?}", coordinator_realm_state.checkpoint_id, self.db.state.last_committed_realm_end_root, - coordinator_realm_state.value, self.proposals.len()); - return self.db.sync_to_coordinator_checkpoint_id(checkpoint_id).await; + coordinator_realm_state.value); + return self.db.sync_to_coordinator_checkpoint_id(latest_checkpoint_id).await; }; - let included_checkpoint_id = coordinator_realm_state.checkpoint_id; if included_checkpoint_id > self.db.state.last_committed_checkpoint_id { - self.ensure_uncommitted_processing_ids(included_checkpoint_id).await?; - let coordinator_update = self.db.coordinator_client - .rc_get_realm_sync_info(included_checkpoint_id, self.db.state.realm_id_u64) - .await?; - self.db.state.processing_checkpoint_id = included_checkpoint_id; - self.db.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; - self.db.state.processing_realm_start_root = updates.old_realm_root; - self.db.state.processing_realm_end_root = updates.new_realm_root; - self.db.commit_state( - &coordinator_update, &updates, ProvingJobCircuitType::GUTANoChange, vec![], true, - ).await?; - tracing::info!("Committed Realm proposal {} FFS checkpoint_id={}", hex::encode(proposal_id), included_checkpoint_id); + tracing::info!( + "Committed Realm proposal FFS checkpoint_id={}", + included_checkpoint_id + ); } self.db.state.gathering_realm_start_root = updates.new_realm_root; self.db.shared_state.update_from_core_state(&self.db.state).await?; - // The author's gatherer already contains this transition. - if updates.realm_sub_id != self.db.state.realm_sub_id_u64 { + if gathering_start != updates.old_realm_root && gathering_start != updates.new_realm_root { + self.rebuild_production_gatherer().await?; + } else if updates.realm_sub_id != self.db.state.realm_sub_id_u64 { self.guta_queue_gatherer.fast_forward(updates_bytes).await?; } - self.proposals.remove(&proposal_id); - // Bound metadata advancement to the applied root's checkpoint. A newer - // transition is selected on the next sync rather than skipped as metadata. + self.db + .publish_validator_leaves(self.p2p.as_ref(), self.db.state.last_committed_checkpoint_id) + .await?; self.db.sync_to_coordinator_checkpoint_id(self.db.state.last_committed_checkpoint_id).await } @@ -345,7 +417,10 @@ where - pub async fn process_block(&mut self) -> anyhow::Result<()> { + pub async fn process_block(&mut self) -> anyhow::Result<()> + where + N::HasherBase: MerkleZeroHasher, + { self.db.run_sanity_check("process_block start").await?; let mut timer = TraceTimer::new("process_block"); tracing::info!( @@ -526,11 +601,7 @@ where .await?; } - if let Some((proposal, certificate, output_bytes, worker_tag)) = p2p_submission.as_ref() { - let finalize_binding = RealmFinalizeBinding { - output: *output_bytes, - finalizer_worker_reward_tag: *worker_tag, - }; + if let Some((proposal, certificate, _output_bytes, _worker_tag)) = p2p_submission.as_ref() { tracing::info!( "Submitting GUTA proof to Coordinator proposal={} realm={} sub_id={}", hex::encode(proposal.proposal_id), @@ -545,7 +616,6 @@ where self.db.state.realm_id_u64, Some(proposal.protocol_encode_to_vec()), Some(certificate.protocol_encode_to_vec()), - finalize_binding.protocol_encode_to_vec(), ) .await?; } else { @@ -558,7 +628,6 @@ where self.db.state.realm_id_u64, None, None, - Vec::new(), ) .await?; } @@ -579,10 +648,18 @@ where // commit_state").await?; self.db - .commit_state(&sync_info, &db_output, root_job_id.circuit_type, root_job_proof, false) + .commit_state( + &sync_info, + &db_output, + root_job_id.circuit_type, + root_job_proof, + ) .await?; timer.lap("commit_state"); self.db.run_sanity_check("after commit").await?; + self.db + .publish_validator_leaves(self.p2p.as_ref(), self.db.state.last_committed_checkpoint_id) + .await?; tracing::info!( "Committed new realm block with checkpoint_id = {}.", @@ -648,7 +725,7 @@ where .ok_or_else(|| anyhow::anyhow!("GUTA Proposal proof-base checkpoint overflow"))?; let local_sub_id = self.db.state.realm_sub_id_u64 as u16; - let (validator_sub_ids, leaf_bls_keys, validator_user_ids) = + let (validator_sub_ids, leaf_bls_keys, validator_user_ids, _) = self.load_base_checkpoint_validators(base_checkpoint_id).await?; let proposer_user_id = validator_user_ids .iter() @@ -703,7 +780,8 @@ where .copied() .filter(|(sub_id, _)| *sub_id != local_sub_id) .collect(); - cmds.publish_proposal(proposal.clone(), body, remote_bls_keys).await?; + self.proposal_store.save_proposal(&proposal, &body).await?; + cmds.publish_proposal(proposal.clone(), body.clone(), remote_bls_keys).await?; cmds.publish_vote(own_vote.clone()).await?; tracing::info!( "realm P2P proposal published proposal={} realm={} sub_id={} epoch={} target={} base={} validator_tree_root={}", @@ -721,7 +799,6 @@ where let deadline = tokio::time::Instant::now() + Duration::from_secs(120); while !votes_meet_wait( n, - local_sub_id, &all_votes.iter().map(|(sub_id, _)| *sub_id).collect::>(), ) { let remaining_time = deadline.saturating_duration_since(tokio::time::Instant::now()); @@ -769,6 +846,7 @@ where } let certificate = form_certificate(&proposal, &all_votes)?; validate_certificate(&proposal, &certificate, &validator_sub_ids, &leaf_bls_keys)?; + self.proposal_store.save_proposal(&proposal, &body).await?; let signer_ids = all_votes.iter().map(|(sub_id, _)| *sub_id).collect::>(); tracing::info!( "realm P2P certificate formed proposal={} realm={} target={} epoch={} signers={:?} verified_votes={}", @@ -789,6 +867,7 @@ where Vec, Vec<(u16, psy_data::p2p::BlsPublicKey)>, Vec<(u16, u64)>, + Vec<(u16, psy_data::p2p::ValidatorLeaf)>, )> where N::HasherBase: MerkleZeroHasher, @@ -814,7 +893,7 @@ where let target = base_checkpoint_id .checked_add(1) .ok_or_else(|| anyhow::anyhow!("GUTA Proposal proof-base checkpoint overflow"))?; - let (validator_sub_ids, _, _) = self.load_base_checkpoint_validators(base_checkpoint_id).await?; + let (validator_sub_ids, _, _, _) = self.load_base_checkpoint_validators(base_checkpoint_id).await?; let tree_rotation = parth_common::realm_rotation::RealmRotationConfig { checkpoints_per_epoch: CHECKPOINTS_PER_EPOCH, validator_sub_ids, diff --git a/psy_node_common/src/realm/processor/core/runner.rs b/psy_node_common/src/realm/processor/core/runner.rs index 8a8949e8..2eecac76 100644 --- a/psy_node_common/src/realm/processor/core/runner.rs +++ b/psy_node_common/src/realm/processor/core/runner.rs @@ -1,7 +1,7 @@ use cf_utils::log_indicator::print_cf_log_indicator; use parth_core::{ - crypto::hash::traits::HashTo4Felts, + crypto::hash::traits::{HashTo4Felts, MerkleZeroHasher}, felt::ToU64Value, protocol::core_types::QNetworkTypesConfig, }; @@ -15,11 +15,30 @@ use psy_node_core::{ }; use tokio::time::sleep; -use crate::{p2p::guta_submit::GutaSubmitError, realm::processor::core::PsyRealmProcessor}; +use crate::{p2p::guta_submit::GutaSubmitError, queue::gatherer::GathererChannelClosed, realm::processor::core::PsyRealmProcessor, utils::processor_status::ProcessorStatus}; + +async fn join_gatherer( + handle: &mut Option>>, + status: &ProcessorStatus, +) -> anyhow::Result { + let handle = handle.take().ok_or_else(|| anyhow::anyhow!("GUTA gatherer handle missing"))?; + match handle.await { + Ok(result) => result?, + Err(error) if error.is_cancelled() && matches!(status.state(), crate::utils::processor_status::ProcessorState::Stopping | crate::utils::processor_status::ProcessorState::Stopped) => return Ok(false), + Err(error) => return Err(anyhow::Error::new(error).context("GUTA gatherer task failed")), + } + Ok(status.should_run()) +} + +fn report_processor_failure(realm_id: u64, realm_sub_id: u64, error: &anyhow::Error) { + let cause = format!("{error:#}").replace('\\', "\\\\").replace('\r', "\\r").replace('\n', "\\n"); + eprintln!("realm_processor_failure realm_id={realm_id} realm_sub_id={realm_sub_id} error={cause}"); + print_cf_log_indicator("PSY_REALM_PROCESSOR_ERROR", &format!("R{}_{}", realm_id, realm_sub_id)); +} pub async fn run_realm_processor_loop< N: QNetworkTypesConfig, - S: PsyRealmProcessorStore + Send + Sync, + S: PsyRealmProcessorStore + Send + Sync + 'static, STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter + PsyNodeCoreRewardsTagTreeStoreReader + Send + Sync, GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync + 'static, ProofWorkQueue: QStandardWorkerQueuePublisher + QStandardWorkerQueueSubscriber + Send + Sync + 'static, @@ -41,7 +60,9 @@ pub async fn run_realm_processor_loop< >, ) -> anyhow::Result<()> where - N: 'static, FileSystem::File: Send + Sync + 'static, + N: 'static, + N::HasherBase: MerkleZeroHasher, + FileSystem::File: Send + Sync + 'static, { let realm_id = processor.db.state.realm_id_u64; let realm_sub_id = processor.db.state.realm_sub_id_u64; @@ -50,7 +71,13 @@ where let mut last_slot: u128 = 0; + let result: anyhow::Result<()> = async { loop { + if processor.guta_gatherer_join.as_ref().is_some_and(|handle| handle.is_finished()) { + if join_gatherer(&mut processor.guta_gatherer_join, &processor.db.status).await? { + processor.run_init_catchup().await?; + } + } if processor.db.status.should_run() { // tracing::debug!("[REALM] Sync and verify starting..."); let sync_result = processor.sync_and_verify().await; @@ -84,6 +111,12 @@ where tracing::info!("Generated GUTA Realm update in {}ms at slot {}", duration_ms, current_slot); } Err(e) => { + if e.downcast_ref::().is_some() { + if join_gatherer(&mut processor.guta_gatherer_join, &processor.db.status).await? { + processor.run_init_catchup().await?; + } + continue; + } if e.downcast_ref::().is_some_and(|submit| submit.is_retryable()) { tracing::warn!( "[REALM] Retryable process_block rejection at slot {} after {}ms: {:#}", @@ -99,10 +132,21 @@ where ); } } else { - let error = format!("realm process_block failed at slot {}: {:#}", current_slot, e); - processor.db.status.require_recovery(error.clone()); - tracing::error!("[REALM] Fatal error processing block: {:?}, took {}ms at slot {}; processor parked in Error state until manually restarted", e, duration_ms, current_slot); - print_cf_log_indicator("PSY_REALM_PROCESSOR_ERROR", &format!("R{}_{}", realm_id, realm_sub_id)); + tracing::error!( + "[REALM] Fatal error processing block: {:?}, took {}ms at slot {}; aborting gatherer and re-entering init catch-up", + e, + duration_ms, + current_slot + ); + report_processor_failure(realm_id, realm_sub_id, &e); + if let Err(error) = processor.run_init_catchup().await { + tracing::error!( + "init catch-up after fatal process_block failed sub_id={} error={:#}", + realm_sub_id, + error + ); + sleep(std::time::Duration::from_secs(5)).await; + } } } } @@ -116,10 +160,17 @@ where break; } } + processor.abort_production_gatherer().await; processor.db.status.mark_stopped(); print_cf_log_indicator("PSY_REALM_PROCESSOR_STOPPED", &format!("R{}_{}", realm_id, realm_sub_id)); Ok(()) + }.await; + if let Err(error) = &result { + processor.db.status.require_recovery(format!("{error:#}")); + report_processor_failure(realm_id, realm_sub_id, error); + } + result } pub async fn run_realm_processor< N: QNetworkTypesConfig, @@ -143,10 +194,10 @@ pub async fn run_realm_processor< FileSystem, CoordinatorClient, >, - guta_gatherer_join_handle: tokio::task::JoinHandle>, ) -> anyhow::Result<()> where N: 'static, + N::HasherBase: MerkleZeroHasher, FileSystem::File: Send + Sync + 'static, { let status = processor.db.status.clone(); @@ -158,52 +209,77 @@ where sleep(std::time::Duration::from_secs(5)).await; Ok(()) } - result = async { - let mut tasks: Vec<(&'static str, tokio::task::JoinHandle>)> = vec![ - ("realm processor", tokio::spawn(run_realm_processor_loop(processor))), - ("GUTA gatherer", guta_gatherer_join_handle), - ]; - - // Observe tasks directly. The first failure marks the processor - // status Error, aborts and joins only the still-running tasks so - // no live mutation is left detached, and returns the original - // error. Completed entries are removed before cleanup so a - // finished handle is never awaited twice. - let outcome = loop { - if tasks.is_empty() { - break Ok::<(), anyhow::Error>(()); - } - let boxed: Vec<_> = tasks - .iter_mut() - .map(|(name, handle)| Box::pin(async move { (*name, handle.await) })) - .collect(); - let ((name, completion), index, rest) = futures::future::select_all(boxed).await; - drop(rest); - tasks.swap_remove(index); - let error = match completion { - Ok(Ok(())) => continue, - Ok(Err(error)) => error, - Err(join_error) => anyhow::Error::new(join_error).context(format!("{name} task panicked")), - }; - status.require_recovery(format!("{name} failed: {error:#}")); - for (name, handle) in tasks.iter_mut() { - handle.abort(); - match handle.await { - Ok(Ok(())) => {} - Ok(Err(error)) => tracing::error!("{} task failed during cleanup: {:#}", *name, error), - Err(join_error) if !join_error.is_cancelled() => { - tracing::error!("{} task join error during cleanup: {}", *name, join_error); - } - Err(_) => {} - } - } - break Err(error); - }; - outcome - } => { + result = run_realm_processor_loop(processor) => { result?; tracing::info!("All realm processor threads completed"); Ok(()) } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, thiserror::Error)] + #[error("validator preimage mismatch")] + struct ProofFailure; + + #[tokio::test] + async fn channel_close_waits_for_owner_and_preserves_failure_during_shutdown() { + let status = ProcessorStatus::new(); + status.mark_running(); + let (closed_tx, closed_rx) = tokio::sync::oneshot::channel::<()>(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mut handle = Some(tokio::spawn(async move { + drop(closed_tx); + release_rx.await.unwrap(); + Err(anyhow::Error::new(ProofFailure).context("gatherer bootstrap")) + })); + closed_rx.await.unwrap_err(); + assert!(!handle.as_ref().unwrap().is_finished()); + status.begin_shutdown(); + let mut joined = Box::pin(join_gatherer(&mut handle, &status)); + tokio::select! { + biased; + result = &mut joined => panic!("owner has not exited: {result:?}"), + _ = std::future::ready(()) => {} + } + release_tx.send(()).unwrap(); + let error = joined.await.unwrap_err(); + assert!(error.downcast_ref::().is_some()); + assert!(handle.is_none()); + } + + #[tokio::test] + async fn clean_owner_exit_rebuilds_only_while_running() { + let status = ProcessorStatus::new(); + status.mark_running(); + let mut handle = Some(tokio::spawn(async { Ok(()) })); + assert!(join_gatherer(&mut handle, &status).await.unwrap()); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let task_status = status.clone(); + let mut handle = Some(tokio::spawn(async move { + release_rx.await.unwrap(); + task_status.begin_shutdown(); + Ok(()) + })); + release_tx.send(()).unwrap(); + assert!(!join_gatherer(&mut handle, &status).await.unwrap()); + assert_eq!(status.state(), crate::utils::processor_status::ProcessorState::Stopping); + assert!(status.error().is_none()); + } + + #[tokio::test] + async fn cancelled_owner_is_expected_only_during_shutdown() { + let status = ProcessorStatus::new(); + status.mark_running(); + let task = tokio::spawn(std::future::pending::>()); + task.abort(); + assert!(join_gatherer(&mut Some(task), &status).await.unwrap_err().downcast_ref::().unwrap().is_cancelled()); + status.begin_shutdown(); + let task = tokio::spawn(std::future::pending::>()); + task.abort(); + assert!(!join_gatherer(&mut Some(task), &status).await.unwrap()); + } +} diff --git a/psy_node_common/src/realm/processor/core/startup.rs b/psy_node_common/src/realm/processor/core/startup.rs index 1a87c2f3..747e8972 100644 --- a/psy_node_common/src/realm/processor/core/startup.rs +++ b/psy_node_common/src/realm/processor/core/startup.rs @@ -1,7 +1,11 @@ use std::sync::Arc; use parth_core::felt::ToU64Value; -use parth_core::{protocol::core_types::QNetworkTypesConfig, QCoreProcCheckpointUniqueId}; +use parth_core::{ + crypto::hash::traits::{FieldQHasher, MerkleZeroHasher}, + protocol::core_types::QNetworkTypesConfig, + QCoreProcCheckpointUniqueId, +}; use psy_core::job::job_id::QProvingJobDataID; use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate; use psy_io::tokio::TokioLikeFileSystem; @@ -36,6 +40,7 @@ impl< > PsyRealmProcessor where FileSystem::File: Send + Sync, + N::HasherBase: MerkleZeroHasher + FieldQHasher, { pub async fn new( mut db: PsyRealmDatabaseProcessor< @@ -52,7 +57,9 @@ where genesis_block_update: PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, file_system: Arc, guta_gatherer_backup_directory: String, - ) -> anyhow::Result<(Self, tokio::task::JoinHandle>)> { + proposal_store: std::sync::Arc, + proposal_fetch: Option, + ) -> anyhow::Result { tracing::info!("[REALM_STARTUP] processor new start"); db.ensure_genesis_applied(genesis_block_update.clone()).await?; tracing::info!("[REALM_STARTUP] ensure_genesis_applied done"); @@ -60,17 +67,29 @@ where .await? .into_tuple(); tracing::info!("[REALM_STARTUP] load_realm_memory_trees_from_db done"); - db.init_with_setup_and_genesis(&file_system, &guta_gatherer_backup_directory, genesis_block_update, &mut global_user_tree) + db.init_with_setup_and_genesis( + &file_system, + &guta_gatherer_backup_directory, + genesis_block_update, + &mut global_user_tree, + proposal_store.as_ref(), + proposal_fetch.as_ref(), + ) .await?; tracing::info!("[REALM_STARTUP] init_with_setup_and_genesis done"); - //db.set_new_unique_ids().await?; + let (global_user_tree,) = load_realm_memory_trees_from_db::( + &db.db, + db.state.last_committed_checkpoint_id, + db.state.realm_id_u64, + ) + .await? + .into_tuple(); + tracing::info!("[REALM_STARTUP] reloaded gatherer tree after catch-up"); tracing::info!("intialized realm processor database, building gatherers..."); - // Fail closed: the shared validator account leaf is loaded from the - // checkpoint-bound validator preimage before any builder is created. let (validator_preimage, _, _) = load_checkpoint_validator::( - &db.db, &db.state, db.state.gathering_checkpoint_id).await?; + &db.db, &db.state, db.state.last_committed_checkpoint_id).await?; let validator_leaf = db.db - .get_user_leaf(db.state.gathering_checkpoint_id, validator_preimage.validator_user_id).await?; + .get_user_leaf(db.state.last_committed_checkpoint_id, validator_preimage.validator_user_id).await?; anyhow::ensure!(validator_leaf.user_id.to_u64_value() == validator_preimage.validator_user_id, "checkpoint validator user leaf belongs to another user"); let guta_create_builder_config = RealmGUTAEndCapGathererConfig:: { @@ -79,7 +98,7 @@ where status: db.shared_state.inner.clone(), temp_db: db.temp_db.clone(), file_system: file_system.clone(), - backup_file_directory: guta_gatherer_backup_directory, + backup_file_directory: guta_gatherer_backup_directory.clone(), coordinator_guta_updates_circuit_whitelist: db.circuit_fingerprint_config.guta_circuit_whitelist_root, checkpoint_tree: db.checkpoint_tree_backup_manager.checkpoint_tree.clone(), future_pending_end_cap_jobs: Arc::new(std::sync::RwLock::new(Vec::new())), @@ -87,7 +106,7 @@ where tree_store: db.db.clone(), _phantom_n: std::marker::PhantomData, }; - let (guta_queue_gatherer, guta_join_handle) = EphemeralQueueGathererWithTree::new_with_status::< + let (guta_queue_gatherer, guta_gatherer_join) = EphemeralQueueGathererWithTree::new_with_status::< GUTAUpdateQueue, RealmGUTAEndCapGathererConfig, N::QHash, @@ -101,20 +120,19 @@ where db.status.clone(), ); - Ok(( - Self { - db, - guta_queue_gatherer: guta_queue_gatherer, - proof_worker_queue_max_time_ms: u64::MAX, - p2p: None, - rotation: None, - bls_secret: None, - proposal_rx: None, - proposals: std::collections::BTreeMap::new(), - }, - - guta_join_handle, - )) + Ok(Self { + db, + guta_queue_gatherer, + proof_worker_queue_max_time_ms: u64::MAX, + p2p: None, + rotation: None, + bls_secret: None, + proposal_store, + baseline_replay_rx: None, + file_system, + guta_gatherer_backup_directory, + guta_gatherer_join: Some(guta_gatherer_join), + }) } /// Wire optional Realm P2P into the processor after construction. @@ -136,6 +154,117 @@ where self.bls_secret = Some(bls_secret); } + pub fn set_baseline_replay_rx( + &mut self, + baseline_replay_rx: tokio::sync::mpsc::Receiver< + crate::realm::processor::recovery::BaselineReplayRequest, + >, + ) { + self.baseline_replay_rx = Some(baseline_replay_rx); + } + + pub async fn abort_production_gatherer(&mut self) { + let Some(handle) = self.guta_gatherer_join.take() else { + return; + }; + handle.abort(); + match handle.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::error!("production gatherer failed during abort: {error:#}"); + } + Err(join_error) if !join_error.is_cancelled() => { + tracing::error!("production gatherer join error during abort: {join_error}"); + } + Err(_) => {} + } + } + + pub async fn run_init_catchup(&mut self) -> anyhow::Result<()> + where + N::HasherBase: parth_core::crypto::hash::traits::MerkleZeroHasher, + { + self.abort_production_gatherer().await; + let (mut global_user_tree,) = load_realm_memory_trees_from_db::( + &self.db.db, + self.db.state.last_committed_checkpoint_id, + self.db.state.realm_id_u64, + ) + .await? + .into_tuple(); + self.db.set_committed_realm_roots_from_db().await?; + self.db + .ensure_backup_restored_if_necessary( + &self.file_system, + &self.guta_gatherer_backup_directory, + &mut global_user_tree, + self.proposal_store.as_ref(), + self.p2p.as_ref(), + ) + .await?; + self.db.sync_to_coordinator_set_checkpoint_id().await?; + self.rebuild_production_gatherer().await + } + + pub async fn rebuild_production_gatherer(&mut self) -> anyhow::Result<()> + where + N::HasherBase: parth_core::crypto::hash::traits::MerkleZeroHasher, + { + self.abort_production_gatherer().await; + let (global_user_tree,) = load_realm_memory_trees_from_db::( + &self.db.db, + self.db.state.last_committed_checkpoint_id, + self.db.state.realm_id_u64, + ) + .await? + .into_tuple(); + let (validator_preimage, _, _) = load_checkpoint_validator::( + &self.db.db, + &self.db.state, + self.db.state.last_committed_checkpoint_id, + ) + .await?; + let validator_leaf = self + .db + .db + .get_user_leaf(self.db.state.last_committed_checkpoint_id, validator_preimage.validator_user_id) + .await?; + anyhow::ensure!( + validator_leaf.user_id.to_u64_value() == validator_preimage.validator_user_id, + "checkpoint validator user leaf belongs to another user" + ); + let guta_create_builder_config = RealmGUTAEndCapGathererConfig:: { + realm_id_u64: self.db.state.realm_id_u64, + realm_sub_id_u64: self.db.state.realm_sub_id_u64, + status: self.db.shared_state.inner.clone(), + temp_db: self.db.temp_db.clone(), + file_system: self.file_system.clone(), + backup_file_directory: self.guta_gatherer_backup_directory.clone(), + coordinator_guta_updates_circuit_whitelist: self.db.circuit_fingerprint_config.guta_circuit_whitelist_root, + checkpoint_tree: self.db.checkpoint_tree_backup_manager.checkpoint_tree.clone(), + future_pending_end_cap_jobs: Arc::new(std::sync::RwLock::new(Vec::new())), + current_validator_user_leaf: Arc::new(std::sync::Mutex::new(validator_leaf)), + tree_store: self.db.db.clone(), + _phantom_n: std::marker::PhantomData, + }; + let (guta_queue_gatherer, guta_gatherer_join) = EphemeralQueueGathererWithTree::new_with_status::< + GUTAUpdateQueue, + RealmGUTAEndCapGathererConfig, + N::QHash, + N::HasherBase, + RealmGUTAEndCapGatherer, + >( + self.db.guta_update_queue.clone(), + guta_create_builder_config, + self.db.guta_queue_key_status_manager.get_queue_key()?, + global_user_tree, + self.db.status.clone(), + ); + self.guta_queue_gatherer = guta_queue_gatherer; + self.guta_gatherer_join = Some(guta_gatherer_join); + Ok(()) + } + pub async fn get_latest_checkpoint_id_internal(&self) -> anyhow::Result { self.db.db.get_latest_checkpoint_id().await diff --git a/psy_node_common/src/realm/processor/create.rs b/psy_node_common/src/realm/processor/create.rs index d69a2ca4..aa732b15 100644 --- a/psy_node_common/src/realm/processor/create.rs +++ b/psy_node_common/src/realm/processor/create.rs @@ -1,6 +1,10 @@ use std::sync::Arc; -use parth_core::{node::realm_identifier::QRealmIdentifier, protocol::core_types::QNetworkTypesConfig}; +use parth_core::{ + crypto::hash::traits::{FieldQHasher, MerkleZeroHasher}, + node::realm_identifier::QRealmIdentifier, + protocol::core_types::QNetworkTypesConfig, +}; use psy_core::job::job_id::QProvingJobDataID; use psy_data::{ config::network_config::PsyNodeCircuitFingerprintConfig, genesis::genesis_block_setup::PsyGenesisBlockSetupData, @@ -40,7 +44,10 @@ pub async fn create_realm_processor< realm_identifier: QRealmIdentifier, circuit_fingerprint_config: PsyNodeCircuitFingerprintConfig, coordinator_client: Arc, -) -> anyhow::Result<( + proof_verifier: Arc, + proposal_store: Arc, + proposal_fetch: Option, +) -> anyhow::Result< PsyRealmProcessor< N, S, @@ -52,10 +59,10 @@ pub async fn create_realm_processor< FileSystem, CoordinatorClient, >, - tokio::task::JoinHandle>, -)> +> where FileSystem::File: Send + Sync, + N::HasherBase: MerkleZeroHasher + FieldQHasher, { tracing::info!("[REALM_CREATE] setup_for_realm start"); let genesis = @@ -77,34 +84,23 @@ where chain_id, realm_identifier, circuit_fingerprint_config, + proof_verifier, file_system.clone(), checkpoint_tree_root_backup_file_path, - genesis.prepared_updates.new_realm_root, - genesis.coordinator_update.checkpoint_sync_info.checkpoint_tree_root, + &genesis, ) .await?; tracing::info!("[REALM_CREATE] db new_init done"); - let processor_result: ( - PsyRealmProcessor< - N, - S, - STagTreeRewards, - GUTAUpdateQueue, - ProofWorkQueue, - TempDatabase, - ProofStore, - FileSystem, - CoordinatorClient, - >, - tokio::task::JoinHandle>, - ) = PsyRealmProcessor::new( + let processor = PsyRealmProcessor::new( db, genesis, file_system, guta_gatherer_backup_directory, + proposal_store, + proposal_fetch, ) .await?; tracing::info!("[REALM_CREATE] processor new done"); - Ok(processor_result) + Ok(processor) } diff --git a/psy_node_common/src/realm/processor/db/commit.rs b/psy_node_common/src/realm/processor/db/commit.rs index f7c98059..43dffb27 100644 --- a/psy_node_common/src/realm/processor/db/commit.rs +++ b/psy_node_common/src/realm/processor/db/commit.rs @@ -1,9 +1,12 @@ -use anyhow::Ok; +use std::collections::HashSet; + +use anyhow::{Context, Ok}; use parth_core::{ QCoreProcCheckpointUniqueId, - crypto::hash:: - merkle_proof::MerkleProofCore - , + crypto::hash::{ + merkle_proof::MerkleProofCore, + traits::{FieldQHasher, MerkleZeroHasher, ZeroableHash}, + }, protocol::core_types::QNetworkTypesConfig, data::queue::queue_key::{PCoreSubjectQueueBase, QPBaseQueueType}, }; @@ -22,7 +25,8 @@ use psy_io::tokio::TokioLikeFileSystem; use psy_node_core::{ p2p::traits::realm_coordinantor::RealmCoordinatorClient, psy_core_db::traits::full::{ - PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, + PsyNodeCheckpointTreeDatabaseReader, PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, + PsyRealmProcessorStore, }, psy_temp_db::StandardProcessorTempDBStoreBase, queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, @@ -35,6 +39,104 @@ use crate::realm::{ queue_key::{RealmUserUpdateQueueKey, RealmProvingWorkQueueKey}, }; +async fn write_checkpoint_state_records( + db: &S, + checkpoint_sync_info: &PQEDCheckpointSyncInfoCompact, + membership: &MerkleProofCore, +) -> anyhow::Result<()> +where + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, +{ + // ORDERING IS LOAD-BEARING: these writes are not transactional. Recovery + // (`get_latest_available_l2_block_state` / `try_get_complete_l2_block_state`) treats a checkpoint as + // complete based on its core metadata records, so the L2 block state MUST be written LAST — after the + // state roots, checkpoint leaf, tree proof, and root mapping. Writing it earlier would let a crash mid-way + // leave a checkpoint that looks complete (L2 present) but is missing its proof/root mapping, which recovery + // would then never backfill. The `latest_l2_block_state` singleton is advanced by the caller + // (`apply_prepared_realm_checkpoint`) only after `set_latest_checkpoint_id`, so it can never lead the committed marker. + db.set_checkpoint_global_state_roots(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.state_roots) + .await?; + db.set_checkpoint_leaf_data(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.checkpoint_leaf) + .await?; + db.checkpoint_tree_injest_merkle_proof(checkpoint_sync_info.checkpoint_id, membership) + .await?; + db.set_checkpoint_root_hash_to_id_mapping(checkpoint_sync_info.checkpoint_tree_root, checkpoint_sync_info.checkpoint_id) + .await?; + db.set_l2_block_state(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.block_state) + .await?; + Ok(()) +} + +async fn apply_realm_ffs_updates( + db: &S, + checkpoint_id: u64, + realm_update: &PsyPreparedRealmBlockStateUpdates, +) -> anyhow::Result<()> +where + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, +{ + if realm_update.update_user_leaves_ffs.is_empty() { + return Ok(()); + } + db.set_user_leaves_ffs(checkpoint_id, &realm_update.update_user_leaves_ffs) + .await?; + db.contract_state_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_contract_state_tree_nodes_ffs) + .await?; + if !realm_update.update_contract_state_imt_leaves_ffs.is_empty() { + db.contract_state_imt_set_leaves_ffs(checkpoint_id, &realm_update.update_contract_state_imt_leaves_ffs) + .await?; + } + db.user_contract_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_user_contract_tree_nodes_ffs) + .await?; + db.global_user_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_global_user_tree_nodes_ffs) + .await?; + Ok(()) +} + +pub(crate) async fn apply_prepared_realm_checkpoint( + db: &S, + coordinator_update: &PsyRealmCoordinatorUpdate, + realm_update: &PsyPreparedRealmBlockStateUpdates, + unique_pending_id: u64, + proc_id: &QCoreProcCheckpointUniqueId, + membership: &MerkleProofCore, +) -> anyhow::Result<()> +where + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, +{ + let checkpoint_id = coordinator_update.checkpoint_sync_info.checkpoint_id; + db.set_unique_pending_id_checkpoint_id_mapping(unique_pending_id, checkpoint_id) + .await?; + db.set_checkpoint_id_to_unique_pending_id_mapping(checkpoint_id, unique_pending_id, proc_id) + .await?; + db.global_user_tree_set_top_tree_merkle_proof(checkpoint_id, &coordinator_update.merkle_proof_to_realm_root) + .await?; + db.set_realm_rewards_tag_tree_top_proof_at_unique_pending_id( + unique_pending_id, + &coordinator_update.reward_tree_top_proof, + ) + .await?; + write_checkpoint_state_records::(db, &coordinator_update.checkpoint_sync_info, membership).await?; + let imt_managed = if checkpoint_id == 0 { + HashSet::new() + } else { + crate::realm::processor::recovery::imt_managed_leaves_from_db::(db, realm_update) + .await? + }; + crate::realm::processor::recovery::require_state_update_record_coverage(realm_update, checkpoint_id, &imt_managed)?; + apply_realm_ffs_updates::(db, checkpoint_id, realm_update).await?; + let durable_tip = db.get_latest_checkpoint_id().await?; + if checkpoint_id >= durable_tip { + db.set_latest_checkpoint_id(checkpoint_id).await?; + db.set_l2_latest_block_state(&coordinator_update.checkpoint_sync_info.block_state) + .await?; + } + Ok(()) +} + impl< N: QNetworkTypesConfig, S: PsyRealmProcessorStore + Send + Sync, @@ -47,7 +149,7 @@ impl< CoordinatorClient: RealmCoordinatorClient + Send + Sync, > PsyRealmDatabaseProcessor where - N::HasherBase: 'static + Send + Sync, + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, { pub async fn set_new_unique_ids(&mut self, gathering_realm_end_root: Option) -> anyhow::Result<()> { let (new_gathering_unique_pending_id, new_gathering_proc_checkpoint_unique_id) = self.db.inc_unique_pending_id(1).await?; @@ -122,162 +224,83 @@ where Ok(()) } - pub async fn commit_checkpoint_state_no_guta_update( - &mut self, - checkpoint_sync_info: &PQEDCheckpointSyncInfoCompact, - ) -> anyhow::Result<()> { - let previous = self.write_checkpoint_state_records(checkpoint_sync_info).await?; - - let expected_new_checkpoint_root = previous.compute_root_with_value::(checkpoint_sync_info.checkpoint_leaf_hash); - if expected_new_checkpoint_root != checkpoint_sync_info.checkpoint_tree_root { - anyhow::bail!("Inconsistent checkpoint tree root detected when committing checkpoint ID: {}. Expected root: {:?}, but got: {:?}. This indicates a serious inconsistency in the checkpoint tree state.", - checkpoint_sync_info.checkpoint_id, expected_new_checkpoint_root, checkpoint_sync_info.checkpoint_tree_root); - } - - self.checkpoint_tree_backup_manager - .append_checkpoint_leaf_hash(checkpoint_sync_info.checkpoint_id, checkpoint_sync_info.checkpoint_leaf_hash) - .await?; - - // THIS DOES NOT SET THE LATEST CHECKPOINT ID, THAT MUST BE DONE AT THE VERY END - // OF COMMITTING THE FULL STATE - - Ok(()) - } - - async fn commit_checkpoint_state_after_checkpoint_tree_sync( - &mut self, - checkpoint_sync_info: &PQEDCheckpointSyncInfoCompact, - ) -> anyhow::Result<()> { - self.write_checkpoint_state_records(checkpoint_sync_info).await?; - - // The checkpoint tree backup manager was already synced from coordinator, - // so do not recompute a historical append root or append this leaf again. - Ok(()) - } - - async fn write_checkpoint_state_records( - &mut self, - checkpoint_sync_info: &PQEDCheckpointSyncInfoCompact, - ) -> anyhow::Result> { - let previous: MerkleProofCore = self - .checkpoint_tree_backup_manager - .checkpoint_tree - .get_leaf(checkpoint_sync_info.checkpoint_id); - - // ORDERING IS LOAD-BEARING: these writes are not transactional. Recovery - // (`get_latest_available_l2_block_state` / `try_get_complete_l2_block_state`) treats a checkpoint as - // complete based on its core metadata records, so the L2 block state MUST be written LAST — after the - // state roots, checkpoint leaf, tree proof, and root mapping. Writing it earlier would let a crash mid-way - // leave a checkpoint that looks complete (L2 present) but is missing its proof/root mapping, which recovery - // would then never backfill. The `latest_l2_block_state` singleton is advanced by the caller - // (`commit_state`) only after `set_latest_checkpoint_id`, so it can never lead the committed marker. - self.db - .set_checkpoint_global_state_roots(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.state_roots) - .await?; - self.db - .set_checkpoint_leaf_data(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.checkpoint_leaf) - .await?; - - println!("committing checkpoint proof: {:?}", &previous.to_append_proof::()); - self.db - .checkpoint_tree_injest_merkle_proof(checkpoint_sync_info.checkpoint_id, &previous.to_append_proof::()) - .await?; - - self.db - .set_checkpoint_root_hash_to_id_mapping(checkpoint_sync_info.checkpoint_tree_root, checkpoint_sync_info.checkpoint_id) - .await?; - - // Sentinel write — must remain the final persisted metadata for this checkpoint (see note above). - self.db - .set_l2_block_state(checkpoint_sync_info.checkpoint_id, &checkpoint_sync_info.block_state) - .await?; - - Ok(previous) - } - pub async fn commit_state( &mut self, coordinator_update: &PsyRealmCoordinatorUpdate, realm_update: &PsyPreparedRealmBlockStateUpdates, _state_transition_circuit_type: ProvingJobCircuitType, _zk_proof: Vec, - skip_checkpoint_root_check: bool, ) -> anyhow::Result<()> { let checkpoint_id = coordinator_update.checkpoint_sync_info.checkpoint_id; - let unique_pending_id = self.state.processing_unique_pending_id; - // CRITICAL: set unique_pending_id to checkpoint_id mapping BEFORE ANY OTHER - // STATE UPDATES so we can recover if something goes wrong. - // - // SOLE writer of the (unique_pending_id <-> checkpoint_id) mapping. Catch-up, - // fast-forward, init, and no-jobs-skip paths MUST NOT write this mapping — - // doing so either pollutes it with `processing_unique_pending_id` values that - // were never actually committed, or overwrites a correct entry with a stale - // key -> newer checkpoint pair if the coordinator advanced between commit and - // a subsequent sync. Both break recovery (init.rs:423) and RPC consumers. - self.db - .set_unique_pending_id_checkpoint_id_mapping(unique_pending_id, checkpoint_id) - .await?; - self.db - .set_checkpoint_id_to_unique_pending_id_mapping(checkpoint_id, unique_pending_id, &self.state.processing_proc_checkpoint_unique_id) - .await?; - tracing::info!("Set unique pending ID to checkpoint ID mapping for checkpoint ID: {}", checkpoint_id); - - self.db - .global_user_tree_set_top_tree_merkle_proof(checkpoint_id, &coordinator_update.merkle_proof_to_realm_root) - .await?; - self.db - .set_realm_rewards_tag_tree_top_proof_at_unique_pending_id( - unique_pending_id, - &coordinator_update.reward_tree_top_proof, - ) - .await?; - if skip_checkpoint_root_check { - self.commit_checkpoint_state_after_checkpoint_tree_sync(&coordinator_update.checkpoint_sync_info) - .await?; + let leaf_hash = coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash; + let tree_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + let membership = if checkpoint_id == 0 { + let siblings = (0..N::CHECKPOINT_TREE_HEIGHT as usize) + .map(|level| N::HasherBase::get_zero_hash(level)) + .collect(); + MerkleProofCore::new_from_params::(0, leaf_hash, siblings) + } else { + self.coordinator_client + .rc_get_checkpoint_tree_merkle_proof(checkpoint_id) + .await + .context(format!( + "MissingHistoryProof at C={checkpoint_id}: checkpoint tree membership unavailable" + ))? + }; + anyhow::ensure!( + membership.verify::(), + "MissingHistoryProof at C={checkpoint_id}: checkpoint membership does not verify" + ); + anyhow::ensure!( + membership.index == checkpoint_id, + "MissingHistoryProof at C={checkpoint_id}: membership index {} is not C", + membership.index + ); + anyhow::ensure!( + membership.siblings.len() == N::CHECKPOINT_TREE_HEIGHT as usize, + "MissingHistoryProof at C={checkpoint_id}: membership height {} is not {}", + membership.siblings.len(), + N::CHECKPOINT_TREE_HEIGHT + ); + anyhow::ensure!( + membership.value == leaf_hash, + "MissingHistoryProof at C={checkpoint_id}: membership value does not match checkpoint leaf hash" + ); + anyhow::ensure!( + membership.root == tree_root, + "MissingHistoryProof at C={checkpoint_id}: membership root does not match trusted C after-root" + ); + let trusted_previous_root = if checkpoint_id == 0 { + N::HasherBase::get_zero_hash(N::CHECKPOINT_TREE_HEIGHT as usize) } else { - self.commit_checkpoint_state_no_guta_update(&coordinator_update.checkpoint_sync_info) - .await?; - } - - // START STANDARD STATE UPDATES (technically these can be done in any order - // after the above two are done) start contract updates - if !realm_update.update_user_leaves_ffs.is_empty() { - self.db.set_user_leaves_ffs(checkpoint_id, &realm_update.update_user_leaves_ffs).await?; - tracing::info!("Committed user leaves ffs for checkpoint ID: {}", checkpoint_id); - self.db - .contract_state_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_contract_state_tree_nodes_ffs) - .await?; - tracing::info!("Committed contract state tree updates for checkpoint ID: {}", checkpoint_id); - // Write IMT (Indexed Merkle Tree) leaf preimages and key index entries - if !realm_update.update_contract_state_imt_leaves_ffs.is_empty() { - self.db - .contract_state_imt_set_leaves_ffs(checkpoint_id, &realm_update.update_contract_state_imt_leaves_ffs) - .await?; - tracing::info!("Committed contract state IMT leaf updates for checkpoint ID: {}", checkpoint_id); - } - self.db - .user_contract_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_user_contract_tree_nodes_ffs) - .await?; - tracing::info!("Committed user contract tree updates for checkpoint ID: {}", checkpoint_id); self.db - .global_user_tree_set_nodes_ffs(checkpoint_id, &realm_update.update_global_user_tree_nodes_ffs) - .await?; - tracing::info!("Committed global user tree updates for checkpoint ID: {}", checkpoint_id); - } - // END STANDARD STATE UPDATES (technically these can be done in any order after - // the above two are done) + .checkpoint_tree_get_root_hash(checkpoint_id - 1) + .await + .context(format!( + "MissingHistoryProof at C={checkpoint_id}: trusted C-1 checkpoint root read failed" + ))? + }; + anyhow::ensure!( + membership.compute_root_with_value::(N::QHash::get_zero_value()) + == trusted_previous_root, + "MissingHistoryProof at C={checkpoint_id}: empty-leaf root does not match trusted C-1 root" + ); + coordinator_update + .checkpoint_sync_info + .ensure_valid::(&membership.siblings)?; - // CRITICAL: we need to set the checkpoint id at the VERY END otherwise the - // recovery doesn't work this enables us to avoid having to do atomic - // commits, since if the node dies during this process, it will load the backups - // from disk SO LONG AS THE checkpoint_id is not set!!!! + let unique_pending_id = self.state.processing_unique_pending_id; + apply_prepared_realm_checkpoint::( + self.db.as_ref(), + coordinator_update, + realm_update, + unique_pending_id, + &self.state.processing_proc_checkpoint_unique_id, + &membership, + ) + .await?; + tracing::info!("Set unique pending ID to checkpoint ID mapping for checkpoint ID: {}", checkpoint_id); let previous_checkpoint_id = self.state.last_committed_checkpoint_id; - self.db.set_latest_checkpoint_id(checkpoint_id).await?; - // Advance the `latest_l2_block_state` singleton only AFTER the checkpoint marker is committed, so the RPC - // `get_latest_l2_block_state` can never expose a block state that leads the committed `latest_checkpoint_id`. - self.db - .set_l2_latest_block_state(&coordinator_update.checkpoint_sync_info.block_state) - .await?; if checkpoint_id > 0 && previous_checkpoint_id < checkpoint_id { if let Some((previous_pending_id, _)) = self .db @@ -311,3 +334,49 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use parth_common::memory_stores::dash_tree_append_only::PsyDashMemoryAppendOnlyMerkleStore; + use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; + use parth_core::crypto::hash::merkle_proof::MerkleProofCore; + use parth_core::crypto::hash::traits::{FromU64x4, MerkleZeroHasher, ZeroableHash}; + use parth_core::pgoldilocks::PoseidonHasher; + use parth_core::PHash; + + #[test] + fn tip_empty_append_proof_is_not_historical_membership() { + let tree = PsyDashMemoryAppendOnlyMerkleStore::::new(8); + tree.set_leaf(0, PHash::from_u64x4([1, 0, 0, 0])); + let tip_empty_append = tree.get_leaf(1).to_append_proof::(); + tree.set_leaf(1, PHash::from_u64x4([2, 0, 0, 0])); + tree.set_leaf(2, PHash::from_u64x4([3, 0, 0, 0])); + let historical = tree.get_historical_merkle_proof_at_historical_index(1, 1); + assert_ne!( + tip_empty_append.value, historical.value, + "commit must not persist get_leaf(C).to_append_proof() taken before the leaf is set" + ); + assert!(historical.verify::()); + assert_eq!(historical.index, 1); + assert_eq!( + historical.compute_root_with_value::(PHash::get_zero_value()), + tree.get_historical_merkle_proof_at_historical_index(0, 0).root + ); + } + + #[test] + fn genesis_setup_membership_has_no_predecessor() { + let height = 8usize; + let leaf = PHash::from_u64x4([9, 0, 0, 0]); + let siblings = (0..height) + .map(|level| PoseidonHasher::get_zero_hash(level)) + .collect(); + let membership = MerkleProofCore::new_from_params::(0, leaf, siblings); + assert!(membership.verify::()); + assert_eq!(membership.index, 0); + assert_eq!( + membership.compute_root_with_value::(PHash::get_zero_value()), + PoseidonHasher::get_zero_hash(height) + ); + } +} diff --git a/psy_node_common/src/realm/processor/db/core.rs b/psy_node_common/src/realm/processor/db/core.rs index 9ee83bc6..f08383a9 100644 --- a/psy_node_common/src/realm/processor/db/core.rs +++ b/psy_node_common/src/realm/processor/db/core.rs @@ -88,6 +88,7 @@ pub struct PsyRealmDatabaseProcessor< // config pub circuit_fingerprint_config: PsyNodeCircuitFingerprintConfig, + pub proof_verifier: Arc, } impl< @@ -148,6 +149,29 @@ where .await?; Ok(realm_root_hash) } + + pub async fn set_last_committed_realm_root_from_db(&mut self) -> anyhow::Result<()> { + let current_realm_root = self + .db + .global_user_tree_get_node(self.state.last_committed_checkpoint_id, self.realm_root_node) + .await?; + self.state.last_committed_realm_end_root = current_realm_root; + self.state.last_committed_realm_start_root = current_realm_root; + self.shared_state.update_from_core_state(&self.state).await + } + + pub async fn set_committed_realm_roots_from_db(&mut self) -> anyhow::Result<()> { + let current_realm_root = self + .db + .global_user_tree_get_node(self.state.last_committed_checkpoint_id, self.realm_root_node) + .await?; + self.state.last_committed_realm_end_root = current_realm_root; + self.state.last_committed_realm_start_root = current_realm_root; + self.state.processing_realm_start_root = current_realm_root; + self.state.processing_realm_end_root = current_realm_root; + self.state.gathering_realm_start_root = current_realm_root; + self.shared_state.update_from_core_state(&self.state).await + } } impl< diff --git a/psy_node_common/src/realm/processor/db/genesis.rs b/psy_node_common/src/realm/processor/db/genesis.rs new file mode 100644 index 00000000..ac0458d7 --- /dev/null +++ b/psy_node_common/src/realm/processor/db/genesis.rs @@ -0,0 +1,359 @@ +use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; +use parth_core::{ + QCoreProcCheckpointUniqueId, + crypto::hash::{ + merkle_proof::MerkleProofCore, + traits::{FieldQHasher, MerkleZeroHasher, ZeroableHash}, + }, + protocol::core_types::{Q256BitHash, QNetworkTypesConfig}, +}; +use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate; +use psy_io::tokio::TokioLikeFileSystem; +use psy_node_core::psy_core_db::traits::full::PsyRealmProcessorStore; + +use crate::{ + backup::checkpoint_tree::CheckpointTreeBackupManager, + realm::processor::db::{commit::apply_prepared_realm_checkpoint, DatabaseCheckState}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GenesisBootstrapPlan { + pub write_checkpoint_zero: bool, + pub seed_backup_checkpoint_zero: bool, + pub write_validators: bool, + pub write_complete: bool, +} + +impl GenesisBootstrapPlan { + pub const SKIP: Self = Self { + write_checkpoint_zero: false, + seed_backup_checkpoint_zero: false, + write_validators: false, + write_complete: false, + }; +} + +pub fn plan_genesis_bootstrap( + latest_checkpoint_id: u64, + next_backup_checkpoint_id: u64, + checkpoint_zero_mapping_missing: bool, + checkpoint_zero_l2_incomplete: bool, +) -> GenesisBootstrapPlan { + if latest_checkpoint_id > 0 { + return GenesisBootstrapPlan::SKIP; + } + GenesisBootstrapPlan { + write_checkpoint_zero: checkpoint_zero_mapping_missing || checkpoint_zero_l2_incomplete, + seed_backup_checkpoint_zero: next_backup_checkpoint_id == 0, + write_validators: true, + write_complete: true, + } +} + +pub fn should_recover_cleared_backup(local_tip: u64, next_backup_checkpoint_id: u64) -> bool { + next_backup_checkpoint_id == 0 && local_tip > 0 +} + +pub fn should_hard_reset_ahead_backup(coordinator_tip: u64, next_backup_checkpoint_id: u64) -> bool { + coordinator_tip == 0 && next_backup_checkpoint_id > 0 +} + +pub async fn seed_or_check_genesis_backup( + manager: &mut CheckpointTreeBackupManager, + trusted_leaf: Hash, +) -> anyhow::Result +where + Hasher: MerkleZeroHasher + Send + Sync + 'static, + Hash: Eq + Copy + PartialEq + Default + std::hash::Hash + Q256BitHash, + FileSystem: TokioLikeFileSystem, +{ + if manager.next_backup_checkpoint_id == 0 { + manager.append_checkpoint_leaf_hash(0, trusted_leaf).await?; + return Ok(true); + } + if manager.min_backed_up_checkpoint_id == 0 { + let existing = manager.checkpoint_tree.get_leaf_value(0); + anyhow::ensure!( + existing == trusted_leaf, + "backup checkpoint 0 leaf does not match trusted genesis hash" + ); + } + Ok(false) +} + +pub fn classify_genesis_mapping( + local_latest_checkpoint_id: u64, + checkpoint_zero_pending: anyhow::Result>, +) -> anyhow::Result> { + if local_latest_checkpoint_id == 0 && checkpoint_zero_pending?.is_none() { + return Ok(Some(DatabaseCheckState::NeedsGenesis)); + } + Ok(None) +} + +pub fn classify_genesis_complete_gate( + genesis_complete: bool, + local_latest_checkpoint_id: u64, + checkpoint_zero_pending: anyhow::Result>, +) -> anyhow::Result> { + if genesis_complete { + if classify_genesis_mapping(local_latest_checkpoint_id, checkpoint_zero_pending)?.is_some() { + anyhow::bail!("genesis complete record exists but checkpoint 0 mapping is missing"); + } + return Ok(None); + } + Ok(Some(DatabaseCheckState::NeedsGenesis)) +} + +pub async fn apply_genesis_checkpoint_records( + db: &S, + genesis: &PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, + unique_pending_id: u64, + proc_id: QCoreProcCheckpointUniqueId, +) -> anyhow::Result<()> +where + N: QNetworkTypesConfig, + N::HasherBase: MerkleZeroHasher + FieldQHasher, + S: PsyRealmProcessorStore + Send + Sync, +{ + let checkpoint_id = genesis.coordinator_update.checkpoint_sync_info.checkpoint_id; + anyhow::ensure!( + checkpoint_id == 0, + "genesis checkpoint write requires checkpoint 0, got {checkpoint_id}" + ); + let leaf_hash = genesis.coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash; + let tree_root = genesis.coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + let siblings = (0..N::CHECKPOINT_TREE_HEIGHT as usize) + .map(|level| N::HasherBase::get_zero_hash(level)) + .collect(); + let membership = MerkleProofCore::new_from_params::(0, leaf_hash, siblings); + anyhow::ensure!( + membership.verify::(), + "genesis checkpoint 0 membership does not verify" + ); + anyhow::ensure!( + membership.value == leaf_hash && membership.root == tree_root, + "genesis checkpoint 0 membership does not match the trusted bundle" + ); + anyhow::ensure!( + membership.compute_root_with_value::(N::QHash::get_zero_value()) + == N::HasherBase::get_zero_hash(N::CHECKPOINT_TREE_HEIGHT as usize), + "genesis checkpoint 0 empty-leaf root is not the empty checkpoint tree" + ); + genesis + .coordinator_update + .checkpoint_sync_info + .ensure_valid::(&membership.siblings)?; + apply_prepared_realm_checkpoint::( + db, + &genesis.coordinator_update, + &genesis.prepared_updates, + unique_pending_id, + &proc_id, + &membership, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::{ + classify_genesis_complete_gate, classify_genesis_mapping, plan_genesis_bootstrap, GenesisBootstrapPlan, + }; + use crate::realm::processor::db::DatabaseCheckState; + + #[derive(Default)] + struct GenesisApplyLog { + checkpoint_zero_writes: u32, + backup_zero_appends: u32, + validator_writes: u32, + complete_writes: u32, + } + + fn apply_plan(log: &mut GenesisApplyLog, plan: GenesisBootstrapPlan) { + if plan.write_checkpoint_zero { + log.checkpoint_zero_writes += 1; + } + if plan.seed_backup_checkpoint_zero { + log.backup_zero_appends += 1; + } + if plan.write_validators { + log.validator_writes += 1; + } + if plan.write_complete { + log.complete_writes += 1; + } + } + + #[test] + fn fresh_genesis_bootstraps_complete_db_and_backup_once() { + let first = plan_genesis_bootstrap(0, 0, true, true); + assert_eq!( + first, + GenesisBootstrapPlan { + write_checkpoint_zero: true, + seed_backup_checkpoint_zero: true, + write_validators: true, + write_complete: true, + } + ); + let mut log = GenesisApplyLog::default(); + apply_plan(&mut log, first); + apply_plan(&mut log, plan_genesis_bootstrap(0, 1, false, false)); + assert_eq!(log.checkpoint_zero_writes, 1); + assert_eq!(log.backup_zero_appends, 1); + assert_eq!(log.validator_writes, 2); + assert_eq!(log.complete_writes, 2); + } + + #[test] + fn restart_after_complete_genesis_does_not_reapply_zero_pending_id() { + assert_eq!( + classify_genesis_complete_gate(true, 0, Ok(Some((0, 0)))).expect("complete genesis"), + None + ); + assert_eq!(plan_genesis_bootstrap(0, 1, false, false).write_checkpoint_zero, false); + assert_eq!( + classify_genesis_mapping(0, Ok(Some((0, 0)))).expect("applied mapping"), + None + ); + assert_eq!(plan_genesis_bootstrap(1, 2, true, true), GenesisBootstrapPlan::SKIP); + } + + #[test] + fn crash_mid_genesis_restart_finishes_before_recovery() { + assert_eq!( + classify_genesis_complete_gate(false, 0, Ok(Some((0, 0)))).expect("mapping is not complete"), + Some(DatabaseCheckState::NeedsGenesis) + ); + let after_mapping_before_l2 = plan_genesis_bootstrap(0, 1, false, true); + assert!(after_mapping_before_l2.write_checkpoint_zero); + assert!(!after_mapping_before_l2.seed_backup_checkpoint_zero); + assert!(after_mapping_before_l2.write_validators); + assert!(after_mapping_before_l2.write_complete); + } + + #[test] + fn backup_ahead_of_zero_tip_restart_preserves_history_and_skips_genesis_append() { + let plan = plan_genesis_bootstrap(0, 8, false, false); + assert!(!plan.write_checkpoint_zero); + assert!(!plan.seed_backup_checkpoint_zero); + assert!(plan.write_validators); + assert!(plan.write_complete); + } + + #[test] + fn mapping_only_or_incomplete_l2_replays_c0_without_append() { + let mapping_only = plan_genesis_bootstrap(0, 1, false, true); + assert!(mapping_only.write_checkpoint_zero); + assert!(!mapping_only.seed_backup_checkpoint_zero); + let mapping_missing = plan_genesis_bootstrap(0, 1, true, false); + assert!(mapping_missing.write_checkpoint_zero); + assert!(!mapping_missing.seed_backup_checkpoint_zero); + } + + #[test] + fn cleared_backup_with_positive_tip_triggers_recovery() { + assert!(super::should_recover_cleared_backup(8, 0)); + assert!(!super::should_recover_cleared_backup(0, 0)); + assert!(!super::should_recover_cleared_backup(8, 8)); + } + + #[test] + fn coordinator_at_zero_hard_resets_ahead_backup() { + assert!(super::should_hard_reset_ahead_backup(0, 8)); + assert!(!super::should_hard_reset_ahead_backup(0, 0)); + assert!(!super::should_hard_reset_ahead_backup(8, 8)); + } + + struct EmptyCheckpointReader; + + #[async_trait::async_trait] + impl psy_node_core::psy_core_db::traits::full::PsyNodeCheckpointTreeDatabaseReader + for EmptyCheckpointReader + { + async fn checkpoint_tree_get_leaf_hash( + &self, + _checkpoint_id: u64, + _leaf_index: u64, + ) -> anyhow::Result { + anyhow::bail!("empty checkpoint reader has no leaves") + } + async fn checkpoint_tree_get_root_hash(&self, _checkpoint_id: u64) -> anyhow::Result { + anyhow::bail!("empty checkpoint reader has no roots") + } + async fn checkpoint_tree_get_merkle_proof( + &self, + _checkpoint_id: u64, + _leaf_index: u64, + ) -> anyhow::Result> { + anyhow::bail!("empty checkpoint reader has no proofs") + } + async fn checkpoint_tree_get_nodes( + &self, + _checkpoint_id: u64, + _keys: &[parth_core::data::hash::merkle_node_key::SimpleMerkleNodeKey], + ) -> anyhow::Result> { + anyhow::bail!("empty checkpoint reader has no nodes") + } + } + + async fn empty_backup_manager() -> anyhow::Result< + crate::backup::checkpoint_tree::CheckpointTreeBackupManager< + parth_core::pgoldilocks::PoseidonHasher, + parth_core::PHash, + psy_node_core::file::memory_fs::SimpleMockMemoryFileSystem, + >, + > { + let file_system = std::sync::Arc::new(psy_node_core::file::memory_fs::SimpleMockMemoryFileSystem::new()); + crate::backup::checkpoint_tree::CheckpointTreeBackupManager::new_from_file_path( + file_system, + 16, + 8, + &EmptyCheckpointReader, + "local_checkpoints/genesis_test.bin", + true, + ) + .await + } + + #[tokio::test] + async fn applied_genesis_and_backup_zero_to_eight_does_not_reappend() -> anyhow::Result<()> { + use parth_core::crypto::hash::traits::FromU64x4; + let mut manager = empty_backup_manager().await?; + let leaf0 = parth_core::PHash::from_u64x4([1, 0, 0, 0]); + assert!(super::seed_or_check_genesis_backup(&mut manager, leaf0).await?); + for checkpoint_id in 1..8 { + manager + .append_checkpoint_leaf_hash(checkpoint_id, parth_core::PHash::from_u64x4([checkpoint_id + 1, 0, 0, 0])) + .await?; + } + assert_eq!(manager.next_backup_checkpoint_id, 8); + assert!(!super::seed_or_check_genesis_backup(&mut manager, leaf0).await?); + assert_eq!(manager.next_backup_checkpoint_id, 8); + let mismatch = super::seed_or_check_genesis_backup(&mut manager, parth_core::PHash::from_u64x4([9, 0, 0, 0])) + .await + .expect_err("mismatched genesis leaf must fail closed"); + assert!(mismatch.to_string().contains("trusted genesis hash"), "{mismatch}"); + Ok(()) + } + + #[tokio::test] + async fn coordinator_at_zero_hard_reset_reseeds_checkpoint_zero() -> anyhow::Result<()> { + use parth_core::crypto::hash::traits::FromU64x4; + let mut manager = empty_backup_manager().await?; + let leaf0 = parth_core::PHash::from_u64x4([1, 0, 0, 0]); + super::seed_or_check_genesis_backup(&mut manager, leaf0).await?; + for checkpoint_id in 1..8 { + manager + .append_checkpoint_leaf_hash(checkpoint_id, parth_core::PHash::from_u64x4([checkpoint_id + 1, 0, 0, 0])) + .await?; + } + assert!(super::should_hard_reset_ahead_backup(0, manager.next_backup_checkpoint_id)); + manager.hard_reset_and_truncate(0).await?; + assert_eq!(manager.next_backup_checkpoint_id, 0); + assert!(super::seed_or_check_genesis_backup(&mut manager, leaf0).await?); + assert_eq!(manager.next_backup_checkpoint_id, 1); + Ok(()) + } +} diff --git a/psy_node_common/src/realm/processor/db/init.rs b/psy_node_common/src/realm/processor/db/init.rs index 500b6b18..5ce250c2 100644 --- a/psy_node_common/src/realm/processor/db/init.rs +++ b/psy_node_common/src/realm/processor/db/init.rs @@ -1,11 +1,13 @@ +use std::collections::HashMap; use std::future::Future; use std::sync::Arc; +use std::time::Duration; use parth_common::memory_stores::{mem_tree_recorder::SimpleMemoryMerkleRecorderStore, traits::PsyMemoryMerkleStoreImm}; use parth_core::{ QCoreProcCheckpointUniqueId, crypto::hash:: - traits::MerkleZeroHasher + traits::{FieldQHasher, MerkleZeroHasher} , data::{ hash::{checkpointed_merkle_node::CheckpointedMerkleHash, merkle_node_key::SimpleMerkleNodeKey}, @@ -28,7 +30,12 @@ use psy_data::{ use psy_io::tokio::TokioLikeFileSystem; use psy_node_core::{ genesis::genesis_db_data_builder::GenesisDatabaseDataBuilder, - p2p::{traits::realm_coordinantor::RealmCoordinatorClient, validator_lookup::write_validator_tree_genesis}, + p2p::{ + traits::realm_coordinantor::RealmCoordinatorClient, + validator_lookup::{ + load_realm_validators_from_tree, validator_nodes_from_leaves, write_validator_tree_genesis, + }, + }, psy_core_db::traits::full::{ PsyNodeCheckpointTreeDatabaseReader, PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, }, @@ -45,7 +52,13 @@ use crate::{ constants::queue::PQ_REALM_SUBMIT_USER_UPDATE_QUEUE_TOPIC_ID, queue::gatherer::QueueKeyStatusManager, realm::processor::{ - db::{DatabaseCheckState, PsyRealmDatabaseProcessor}, + db::{ + genesis::{ + apply_genesis_checkpoint_records, classify_genesis_complete_gate, plan_genesis_bootstrap, + seed_or_check_genesis_backup, should_hard_reset_ahead_backup, should_recover_cleared_backup, + }, + DatabaseCheckState, PsyRealmDatabaseProcessor, + }, gatherers::realm_end_cap_gatherer::{ get_new_realm_end_cap_gatherer_backup_file_path, read_realm_backup_end_root, }, @@ -173,19 +186,19 @@ impl< CoordinatorClient: RealmCoordinatorClient + Send + Sync, > PsyRealmDatabaseProcessor where - N::HasherBase: 'static + Send + Sync, + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, { pub async fn get_database_check_state(&self) -> anyhow::Result { let local_latest_checkpoint_id: u64 = self.db.get_latest_checkpoint_id().await?; - if local_latest_checkpoint_id == 0 { - let (last_unique_pending_id, _) = match self.db.get_latest_mapped_unique_pending_id().await { - Ok(ids) => ids, - Err(_) => return Ok(DatabaseCheckState::NeedsGenesis), - }; - if last_unique_pending_id == 0 { - return Ok(DatabaseCheckState::NeedsGenesis); - } + if classify_genesis_complete_gate( + self.db.get_genesis_complete().await?, + local_latest_checkpoint_id, + self.db.get_unique_pending_id_for_checkpoint_id(0).await, + )? + .is_some() + { + return Ok(DatabaseCheckState::NeedsGenesis); } let coordinator_realm_state: CheckpointedMerkleHash = self @@ -253,10 +266,10 @@ where chain_id: u64, realm_identifier: QRealmIdentifier, circuit_fingerprint_config: PsyNodeCircuitFingerprintConfig, + proof_verifier: Arc, file_system: Arc, checkpoint_tree_root_backup_file_path: String, - genesis_realm_root: N::QHash, - genesis_checkpoint_root: N::QHash, + genesis: &PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, ) -> anyhow::Result { let realm_id_u64 = realm_identifier.realm_id as u64; let realm_sub_id_u64 = realm_identifier.realm_sub_id as u64; @@ -266,60 +279,8 @@ where }; tracing::info!("[REALM_INIT] new_init start"); - let last_committed_checkpoint_id = db.get_latest_checkpoint_id().await?; - tracing::info!("[REALM_INIT] latest checkpoint id = {}", last_committed_checkpoint_id); - let ((current_unique_pending_id, current_core_proc_unique_pending_id), (last_committed_unique_pending_id, last_committed_proc_checkpoint_unique_id)) = - if last_committed_checkpoint_id == 0 { - let committed = match db.get_unique_pending_id_for_checkpoint_id(0).await { - Ok(Some(res)) => res, - _ => (0u64, 0u128), - }; - ((0u64, 0u128), committed) - } else { - resolve_current_and_last_committed_pending_ids( - last_committed_checkpoint_id, - |checkpoint_id| { - let db = db.clone(); - async move { db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await } - }, - || { - let db = db.clone(); - async move { db.get_latest_mapped_unique_pending_id().await } - }, - |unique_pending_id| { - let db = db.clone(); - async move { db.get_checkpoint_id_for_unique_pending_id(unique_pending_id).await } - }, - ) - .await? - }; - tracing::info!( - "[REALM_INIT] current unique ids = ({}, {})", - current_unique_pending_id, - current_core_proc_unique_pending_id - ); - - let last_committed_checkpoint_root = match db.checkpoint_tree_get_root_hash(last_committed_checkpoint_id).await { - Ok(root) => root, - Err(_) if last_committed_checkpoint_id == 0 => genesis_checkpoint_root, - Err(e) => return Err(e), - }; - - let last_committed_realm_root = if last_committed_checkpoint_id == 0 { - genesis_realm_root - } else { - db.global_user_tree_get_node(last_committed_checkpoint_id, realm_root_node).await? - }; - - let state = RealmProcessorCoreState::new_basic( - chain_id, - realm_identifier, - last_committed_checkpoint_id, - last_committed_unique_pending_id, - last_committed_proc_checkpoint_unique_id, - last_committed_checkpoint_root, - last_committed_realm_root, - ); + let (state, current_unique_pending_id, current_core_proc_unique_pending_id) = + Self::initial_realm_state(&db, chain_id, realm_identifier, realm_root_node, genesis).await?; let checkpoint_tree_backup_manager = create_new_checkpoint_backup_manager_from_file_path( file_system.clone(), @@ -359,6 +320,7 @@ where checkpoint_tree_backup_manager, shared_state: RealmProcessorCoreStateWrapper::new(state.clone()), circuit_fingerprint_config, + proof_verifier, guta_queue_key_status_manager: QueueKeyStatusManager::< PQ_REALM_SUBMIT_USER_UPDATE_QUEUE_TOPIC_ID, PsyRealmUserUpdateQueueItem, @@ -376,58 +338,131 @@ where }) } + async fn initial_realm_state( + db: &Arc, + chain_id: u64, + realm_identifier: QRealmIdentifier, + realm_root_node: SimpleMerkleNodeKey, + genesis: &PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, + ) -> anyhow::Result<(RealmProcessorCoreState, u64, QCoreProcCheckpointUniqueId)> { + let last_committed_checkpoint_id = db.get_latest_checkpoint_id().await?; + tracing::info!("[REALM_INIT] latest checkpoint id = {}", last_committed_checkpoint_id); + if !db.get_genesis_complete().await? && last_committed_checkpoint_id == 0 { + apply_genesis_checkpoint_records::(db.as_ref(), genesis, 0, 0).await?; + } + let genesis_checkpoint_root = genesis.coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + let ((current_unique_pending_id, current_core_proc_unique_pending_id), (last_committed_unique_pending_id, last_committed_proc_checkpoint_unique_id)) = + if last_committed_checkpoint_id == 0 { + let committed = match db.get_unique_pending_id_for_checkpoint_id(0).await { + Ok(Some(res)) => res, + _ => (0u64, 0u128), + }; + ((0u64, 0u128), committed) + } else { + resolve_current_and_last_committed_pending_ids( + last_committed_checkpoint_id, + |checkpoint_id| { + let db = db.clone(); + async move { db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await } + }, + || { + let db = db.clone(); + async move { db.get_latest_mapped_unique_pending_id().await } + }, + |unique_pending_id| { + let db = db.clone(); + async move { db.get_checkpoint_id_for_unique_pending_id(unique_pending_id).await } + }, + ).await? + }; + tracing::info!( + "[REALM_INIT] current unique ids = ({}, {})", + current_unique_pending_id, + current_core_proc_unique_pending_id + ); + let last_committed_checkpoint_root = match db.checkpoint_tree_get_root_hash(last_committed_checkpoint_id).await { + Ok(root) => root, + Err(_) if last_committed_checkpoint_id == 0 => genesis_checkpoint_root, + Err(e) => return Err(e), + }; + let last_committed_realm_root = db + .global_user_tree_get_node(last_committed_checkpoint_id, realm_root_node) + .await?; + let state = RealmProcessorCoreState::new_basic( + chain_id, + realm_identifier, + last_committed_checkpoint_id, + last_committed_unique_pending_id, + last_committed_proc_checkpoint_unique_id, + last_committed_checkpoint_root, + last_committed_realm_root, + ); + Ok((state, current_unique_pending_id, current_core_proc_unique_pending_id)) + } + pub async fn ensure_genesis_applied( &mut self, genesis_block_update: PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, ) -> anyhow::Result<()> { - let database_check_state = self.get_database_check_state().await?; - if database_check_state == DatabaseCheckState::NeedsGenesis { - tracing::info!("Applying genesis block setup data to realm processor database..."); - println!("genesis_block_update.coordinator_update: {:?}", genesis_block_update.coordinator_update); - self.checkpoint_tree_backup_manager.append_checkpoint_leaf_hash(0, genesis_block_update.coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash).await?; - self.commit_state( - &genesis_block_update.coordinator_update, - &genesis_block_update.prepared_updates, - ProvingJobCircuitType::GUTANoChange, - vec![], - false, - ) - .await?; - write_validator_tree_genesis( - &*self.db, - &genesis_block_update.update_validator_tree_nodes_ffs, - &genesis_block_update.new_validator_leaf_preimages, - ) - .await?; - tracing::info!("Genesis block setup data applied."); - } - Ok(()) + self.finish_genesis_if_needed(&genesis_block_update).await } pub async fn ensure_genesis_applied_from_setup_data(&mut self, genesis_data: &PsyGenesisBlockSetupData) -> anyhow::Result<()> { - let database_check_state = self.get_database_check_state().await?; - if database_check_state == DatabaseCheckState::NeedsGenesis { - tracing::info!("Applying genesis block setup data to realm processor database..."); - let genesis_block_update = GenesisDatabaseDataBuilder::setup_for_realm::( - &genesis_data, - self.state.chain_id, - self.state.realm_id_u64, - self.state.realm_sub_id_u64, - )?; - self.commit_state( - &genesis_block_update.coordinator_update, - &genesis_block_update.prepared_updates, - ProvingJobCircuitType::GUTANoChange, - vec![], - false, + let genesis_block_update = GenesisDatabaseDataBuilder::setup_for_realm::( + genesis_data, + self.state.chain_id, + self.state.realm_id_u64, + self.state.realm_sub_id_u64, + )?; + self.finish_genesis_if_needed(&genesis_block_update).await + } + + async fn finish_genesis_if_needed( + &mut self, + genesis_block_update: &PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, + ) -> anyhow::Result<()> { + let tip = self.db.get_latest_checkpoint_id().await?; + if tip > 0 { + if !self.db.get_genesis_complete().await? { + self.db.set_genesis_complete().await?; + } + return Ok(()); + } + let mapping_missing = self.db.get_unique_pending_id_for_checkpoint_id(0).await?.is_none(); + let l2_incomplete = self.db.try_get_complete_l2_block_state(0).await?.is_none(); + let plan = plan_genesis_bootstrap( + tip, + self.checkpoint_tree_backup_manager.next_backup_checkpoint_id, + mapping_missing, + l2_incomplete, + ); + seed_or_check_genesis_backup( + &mut self.checkpoint_tree_backup_manager, + genesis_block_update + .coordinator_update + .checkpoint_sync_info + .checkpoint_leaf_hash, + ) + .await?; + if plan.write_checkpoint_zero { + apply_genesis_checkpoint_records::( + self.db.as_ref(), + genesis_block_update, + self.state.processing_unique_pending_id, + self.state.processing_proc_checkpoint_unique_id, ) .await?; + } + if plan.write_validators { write_validator_tree_genesis( &*self.db, &genesis_block_update.update_validator_tree_nodes_ffs, &genesis_block_update.new_validator_leaf_preimages, ) .await?; + } + if plan.write_complete { + self.db.set_genesis_complete().await?; tracing::info!("Genesis block setup data applied."); } Ok(()) @@ -478,286 +513,628 @@ impl< CoordinatorClient: RealmCoordinatorClient + Send + Sync, > PsyRealmDatabaseProcessor where - N::HasherBase: 'static + Send + Sync, + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, { + /// Root pairs for the window: each realm transition at C consumed the previous + /// transition's root, so the pair chain starts at the committed realm end root. + async fn pending_transition_lookups( + &self, + from_checkpoint: u64, + target_tip: u64, + ) -> anyhow::Result> + where + N::QHash: Q256BitHash, + { + let mut needed = Vec::new(); + let mut old_root = self.state.last_committed_realm_end_root.into_owned_32bytes(); + let mut checkpoint_id = from_checkpoint; + while checkpoint_id <= target_tip { + let last_modified = self + .coordinator_client + .rc_get_realm_root_and_last_modified_checkpoint(checkpoint_id, self.state.realm_id_u64) + .await?; + if last_modified.checkpoint_id == checkpoint_id { + let new_root = last_modified.value.into_owned_32bytes(); + if new_root != old_root { + needed.push(psy_data::p2p::RealmTransition { old_root, new_root }); + } + old_root = new_root; + } + checkpoint_id += 1; + } + Ok(needed) + } + + /// Stage every candidate the batch window can supply before the loop starts. + async fn stage_recovery_window( + &self, + from_checkpoint: u64, + target_tip: u64, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + client: &crate::realm::network::RealmNetworkCommands, + validator_nodes: &[(u16, psy_data::p2p::NodeId)], + ) -> anyhow::Result> + where + N::QHash: Q256BitHash, + { + let needed = self + .pending_transition_lookups(from_checkpoint, target_tip) + .await?; + let peers = crate::realm::processor::catchup::CatchupPeers::select( + validator_nodes, + self.state.realm_sub_id_u64 as u16, + )?; + let outcomes = crate::realm::processor::catchup::stage_transition_blocks( + client, + proposal_store, + &peers, + self.state.chain_id, + self.state.realm_id_u64 as u32, + &needed, + &[], + ) + .await; + let mut staged = HashMap::new(); + for outcome in outcomes { + match outcome { + crate::realm::processor::catchup::TransitionFetchOutcome::Staged(pair, staged_proposal) => { + staged.insert(pair, staged_proposal); + } + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => tracing::debug!( + "catch-up window pair=({},{}) not offered by the batch peer", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ), + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => tracing::warn!( + "catch-up window pair=({},{}) failed error={error:#}", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ), + } + } + Ok(staged) + } + + async fn stage_single_transition( + &self, + pair: psy_data::p2p::RealmTransition, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + client: &crate::realm::network::RealmNetworkCommands, + batch_base: u64, + rejected: &[[u8; 32]], + ) -> anyhow::Result> { + let validator_nodes = self.validator_nodes_at(batch_base).await?; + let peers = crate::realm::processor::catchup::CatchupPeers::select( + &validator_nodes, + self.state.realm_sub_id_u64 as u16, + )?; + let outcomes = crate::realm::processor::catchup::stage_transition_blocks( + client, + proposal_store, + &peers, + self.state.chain_id, + self.state.realm_id_u64 as u32, + &[pair], + rejected, + ) + .await; + Ok(outcomes.into_iter().find_map(|outcome| match outcome { + crate::realm::processor::catchup::TransitionFetchOutcome::Staged(_, staged) => Some(staged), + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => { + tracing::debug!( + "catch-up pair=({},{}) not offered by any batch peer", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ); + None + } + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => { + tracing::warn!( + "catch-up pair=({},{}) failed error={error:#}", + hex::encode(pair.old_root), + hex::encode(pair.new_root) + ); + None + } + })) + } + + /// Verify one transition, promote the verified bytes into its record, then apply. + async fn apply_verified_transition( + &mut self, + included: &crate::realm::processor::recovery::CheckpointIdentity, + pair: psy_data::p2p::RealmTransition, + staged: Option, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + rejected: &mut Vec<[u8; 32]>, + ) -> anyhow::Result, Vec)>> { + let verified = match self + .verify_history_transition(included, pair, staged.as_ref(), proposal_store) + .await + { + Ok(verified) => verified, + Err(error) => { + let Some(proposal_id) = crate::realm::processor::recovery::invalid_candidate_id(&error) else { + return Err(error); + }; + rejected.push(proposal_id); + return Ok(None); + } + }; + let Some(verified) = verified else { + return Ok(None); + }; + if let Some(staged) = staged { + proposal_store.install(staged).await?; + } + Ok(Some( + self.apply_history_proposal(included, verified) + .await?, + )) + } + + async fn validator_nodes_at( + &self, + checkpoint_id: u64, + ) -> anyhow::Result> + where + N::HasherBase: MerkleZeroHasher, + { + let roots = self.db.get_checkpoint_global_state_roots(checkpoint_id).await?; + let (_, _, _, leaves) = load_realm_validators_from_tree::( + &*self.db, + self.state.chain_id, + checkpoint_id, + self.state.realm_id_u64 as u32, + &roots.validator_tree_root, + ) + .await?; + Ok(validator_nodes_from_leaves(&leaves)) + } + + pub(crate) async fn publish_validator_leaves( + &self, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, + checkpoint_id: u64, + ) -> anyhow::Result<()> + where + N::HasherBase: MerkleZeroHasher, + { + let Some(client) = proposal_fetch else { + return Ok(()); + }; + let roots = self.db.get_checkpoint_global_state_roots(checkpoint_id).await?; + let (_, _, _, leaves) = load_realm_validators_from_tree::( + &*self.db, + self.state.chain_id, + checkpoint_id, + self.state.realm_id_u64 as u32, + &roots.validator_tree_root, + ) + .await?; + client + .set_validator_leaves(leaves.into_iter().map(|(_, leaf)| leaf).collect()) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + Ok(()) + } + pub async fn ensure_backup_restored_if_necessary( &mut self, file_system: &FileSystem, guta_gatherer_backup_directory: &str, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, ) -> anyhow::Result<()> { let database_check_state = self.get_database_check_state().await?; - if database_check_state == DatabaseCheckState::NeedsRecovery { + let local_tip = self.db.get_latest_checkpoint_id().await?; + let target_tip = self.coordinator_client.rc_get_latest_checkpoint_id().await?; + let next_backup = self.checkpoint_tree_backup_manager.next_backup_checkpoint_id; + if should_hard_reset_ahead_backup(target_tip, next_backup) { + self.checkpoint_tree_backup_manager.hard_reset_and_truncate(0).await?; + let genesis_leaf = self.db.checkpoint_tree_get_leaf_hash(0, 0).await?; + seed_or_check_genesis_backup(&mut self.checkpoint_tree_backup_manager, genesis_leaf).await?; + } + if database_check_state == DatabaseCheckState::NeedsRecovery + || local_tip < target_tip + || should_recover_cleared_backup(local_tip, self.checkpoint_tree_backup_manager.next_backup_checkpoint_id) + { tracing::warn!("Inconsistent Realm Processor State detected. Initiating Recovery."); + self.apply_history_transitions( + file_system, guta_gatherer_backup_directory, global_user_tree, + proposal_store, proposal_fetch, database_check_state, target_tip, + ).await?; + } + Ok(()) + } - let coordinator_latest_checkpoint_id = self.coordinator_client.rc_get_latest_checkpoint_id().await?; - + async fn transition_at( + &mut self, + checkpoint_id: u64, + ) -> anyhow::Result, + psy_data::p2p::RealmTransition, + )>> { + let coordinator_update = self.coordinator_client.rc_get_realm_sync_info(checkpoint_id, self.state.realm_id_u64).await?; + let target_realm_state = self.coordinator_client + .rc_get_realm_root_and_last_modified_checkpoint(checkpoint_id, self.state.realm_id_u64) + .await?; + tracing::info!("Coordinator realm root at checkpoint {}: {:?}", checkpoint_id, target_realm_state.value); + if target_realm_state.value == self.state.last_committed_realm_end_root { + tracing::debug!( + "Checkpoint {}: realm root unchanged ({:?}), skipping recovery.", + checkpoint_id, + target_realm_state.value + ); + return Ok(None); + } + Ok(Some((coordinator_update, psy_data::p2p::RealmTransition { + old_root: self.state.last_committed_realm_end_root.into_owned_32bytes(), + new_root: target_realm_state.value.into_owned_32bytes(), + }))) + } + + async fn apply_history_transitions( + &mut self, + file_system: &FileSystem, + guta_gatherer_backup_directory: &str, + global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, + database_check_state: DatabaseCheckState, + mut target_tip: u64, + ) -> anyhow::Result<()> { + loop { self.checkpoint_tree_backup_manager .sync_from_coordinator_client::(&self.coordinator_client, 2000) .await?; - - let mut checkpoint_id = self.state.last_committed_checkpoint_id + 1; - while checkpoint_id <= coordinator_latest_checkpoint_id { - tracing::info!("Recovering checkpoint {}...", checkpoint_id); - - let coordinator_update = self.coordinator_client.rc_get_realm_sync_info(checkpoint_id, self.state.realm_id_u64).await?; - - let target_realm_state = self - .coordinator_client - .rc_get_realm_root_and_last_modified_checkpoint(checkpoint_id, self.state.realm_id_u64) + // Only checkpoints above the committed tip are visited. + let start = self.db.get_latest_checkpoint_id().await?; + let mut checkpoint_id = start + 1; + let mut rejected_proposal_ids: Vec<[u8; 32]> = Vec::new(); + let mut staged_transitions = HashMap::new(); + if let Some(client) = proposal_fetch { + let validator_nodes = self.validator_nodes_at(start).await?; + staged_transitions = self + .stage_recovery_window(start + 1, target_tip, proposal_store, client, &validator_nodes) .await?; - - tracing::info!("Coordinator realm root at checkpoint {}: {:?}", checkpoint_id, target_realm_state.value); - - if target_realm_state.value == self.state.last_committed_realm_end_root { - tracing::debug!( - "Checkpoint {}: realm root unchanged ({:?}), skipping recovery.", - checkpoint_id, - target_realm_state.value + self.publish_validator_leaves(proposal_fetch, start).await?; + } + while checkpoint_id <= target_tip { + tracing::info!("Recovering checkpoint {}...", checkpoint_id); + let Some((coordinator_update, pair)) = self.transition_at(checkpoint_id).await? else { + // Empty checkpoints still authenticate proof-base roots for a later + // included proposal. Persist C only; do not advance the committed marker. + self.persist_checkpoint_metadata_range(checkpoint_id, checkpoint_id, start) + .await + .map_err(|error| { + anyhow::anyhow!( + "checkpoint metadata sync before skipping unchanged realm root: {error:#}" + ) + })?; + tracing::info!( + "Recovered unchanged Realm checkpoint metadata checkpoint_id={checkpoint_id}" ); checkpoint_id += 1; continue; - } - - self.state.processing_checkpoint_id = checkpoint_id; - self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; - - let prepared_updates = if checkpoint_id == 0 { - tracing::info!("Restore target is checkpoint 0 (genesis); using genesis path without backup file."); - self.state.processing_realm_start_root = target_realm_state.value; - self.state.processing_realm_end_root = target_realm_state.value; - PsyPreparedRealmBlockStateUpdates { - realm_id: self.state.realm_id_u64, - realm_sub_id: self.state.realm_sub_id_u64, - old_realm_root: target_realm_state.value, - new_realm_root: target_realm_state.value, - unique_pending_id: 0, - proc_checkpoint_unique_id: 0, - update_global_user_tree_nodes_ffs: vec![], - update_user_contract_tree_nodes_ffs: vec![], - update_contract_state_tree_nodes_ffs: vec![], - update_user_leaves_ffs: vec![], - update_contract_state_imt_leaves_ffs: vec![], - } - } else { - let realm_pending_id = self - .db - .get_unique_pending_id_for_checkpoint_id(checkpoint_id) - .await?; - let (realm_unique_pending_id, realm_proc_checkpoint_id) = match realm_pending_id { - Some(res) => res, - None => { - let ((current_unique_pending_id, current_proc_checkpoint_id), _) = - resolve_current_and_last_committed_pending_ids( - self.state.last_committed_checkpoint_id, - |checkpoint_id| { - let db = self.db.clone(); - async move { db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await } - }, - || self.db.get_latest_mapped_unique_pending_id(), - |unique_pending_id| self.db.get_checkpoint_id_for_unique_pending_id(unique_pending_id), - ) - .await?; - let last_committed_unique_pending_id = self.state.last_committed_unique_pending_id; - - let mut recovered_from_backup = false; - if current_unique_pending_id > last_committed_unique_pending_id { - for candidate in (last_committed_unique_pending_id + 1)..=current_unique_pending_id { - let path = get_new_realm_end_cap_gatherer_backup_file_path( - guta_gatherer_backup_directory, - self.state.realm_id_u64, - self.state.realm_sub_id_u64, - candidate, - ); - match read_realm_backup_end_root::(file_system, &path.to_string_lossy()).await { - Ok(end_root) if end_root == target_realm_state.value => { - let Some(candidate_proc_checkpoint_id) = - self.db.get_proc_checkpoint_unique_id_for_pending_id(candidate).await? - else { - tracing::warn!( - "Backup pending_id {} matches checkpoint {} end_root but has no durable pending->proc record; skipping instead of borrowing another generation's proc ID.", - candidate, - checkpoint_id - ); - continue; - }; - let mut recovery_state = self.state.clone(); - recovery_state.processing_unique_pending_id = candidate; - recovery_state.processing_proc_checkpoint_unique_id = candidate_proc_checkpoint_id; - recovery_state.processing_realm_start_root = self.state.last_committed_realm_end_root; - recovery_state.processing_realm_end_root = target_realm_state.value; - tracing::info!( - "Found matching backup for checkpoint {}: pending_id={}. Attempting full load.", - checkpoint_id, - candidate - ); - let journal_snapshot = global_user_tree.snapshot(); - match generate_realm_output_from_backups::( - file_system, - guta_gatherer_backup_directory, - &recovery_state, - Some(candidate), - global_user_tree, - ).await { - Ok(updates) if updates.new_realm_root == target_realm_state.value && global_user_tree.get_root() == target_realm_state.value => { - tracing::info!( - "Backup recovery successful for pending_id {}: end_root matches coordinator target {:?}.", - candidate, - target_realm_state.value - ); - self.state.processing_unique_pending_id = recovery_state.processing_unique_pending_id; - self.state.processing_proc_checkpoint_unique_id = - recovery_state.processing_proc_checkpoint_unique_id; - self.state.processing_realm_start_root = - recovery_state.processing_realm_start_root; - self.state.processing_realm_end_root = - recovery_state.processing_realm_end_root; - self.commit_state( - &coordinator_update, - &updates, - ProvingJobCircuitType::GUTANoChange, - vec![], - true, - ).await?; - tracing::info!( - "Checkpoint {} recovered from backup (pending_id={}).", - checkpoint_id, - candidate - ); - recovered_from_backup = true; - break; - } - Ok(updates) => { - global_user_tree.revert_to(journal_snapshot); - tracing::warn!( - "Backup end_root {:?} does not match coordinator target {:?} for pending_id {}. Trying next candidate.", - updates.new_realm_root, - target_realm_state.value, - candidate - ); - } - Err(e) => { - global_user_tree.revert_to(journal_snapshot); - tracing::warn!( - "Backup pending_id {} end_root matches but full load failed: {:?}. Trying next candidate.", - candidate, - e - ); - } - } - } - Ok(end_root) => { - tracing::debug!( - "Backup pending_id {} end_root {:?} does not match coordinator target {:?} for checkpoint {}.", - candidate, - end_root, - target_realm_state.value, - checkpoint_id - ); - } - Err(e) => { - tracing::debug!( - "Failed to read backup pending_id {} for checkpoint {}: {:?}", - candidate, - checkpoint_id, - e - ); - } - } - } - } - - if !recovered_from_backup { - anyhow::bail!( - "Checkpoint {}: realm root changed from {:?} to {:?} but no local backup found. \ - This indicates data loss — the sub-tree nodes required to generate proofs are missing.", - checkpoint_id, - self.state.last_committed_realm_end_root, - target_realm_state.value - ); - } - - let latest_realm_root = self.get_realm_root_from_db().await?; - if latest_realm_root != target_realm_state.value { - anyhow::bail!( - "Post-recovery root mismatch at checkpoint {}! Local: {:?}, Target: {:?}", - checkpoint_id, - latest_realm_root, - target_realm_state.value - ); - } - checkpoint_id += 1; - continue; - } - }; - if realm_unique_pending_id == 0 { - tracing::info!( - "Restore target checkpoint {} maps to unique_pending_id 0 (no backup file); using genesis-like path.", - checkpoint_id - ); - self.state.processing_realm_start_root = target_realm_state.value; - self.state.processing_realm_end_root = target_realm_state.value; - PsyPreparedRealmBlockStateUpdates { - realm_id: self.state.realm_id_u64, - realm_sub_id: self.state.realm_sub_id_u64, - unique_pending_id: 0, - proc_checkpoint_unique_id: realm_proc_checkpoint_id, - old_realm_root: target_realm_state.value, - new_realm_root: target_realm_state.value, - update_global_user_tree_nodes_ffs: vec![], - update_user_contract_tree_nodes_ffs: vec![], - update_contract_state_tree_nodes_ffs: vec![], - update_user_leaves_ffs: vec![], - update_contract_state_imt_leaves_ffs: vec![], - } - } else { - self.state.processing_unique_pending_id = realm_unique_pending_id; - self.state.processing_proc_checkpoint_unique_id = realm_proc_checkpoint_id; - self.state.processing_realm_start_root = self.state.last_committed_realm_end_root; - self.state.processing_realm_end_root = target_realm_state.value; - let prepared = generate_realm_output_from_backups::( - file_system, - guta_gatherer_backup_directory, - &self.state, - Some(realm_unique_pending_id), - global_user_tree, - ) - .await?; - anyhow::ensure!( - global_user_tree.get_root() == target_realm_state.value, - "Checkpoint {}: replayed tree root {:?} does not match coordinator target {:?}.", - checkpoint_id, - global_user_tree.get_root(), - target_realm_state.value - ); - prepared - } }; + if !self.restore_checkpoint_transition( + file_system, guta_gatherer_backup_directory, global_user_tree, + proposal_store, proposal_fetch, database_check_state, target_tip, start, + checkpoint_id, &coordinator_update, pair, &mut staged_transitions, + &mut rejected_proposal_ids, + ).await? { + continue; + } + checkpoint_id += 1; + } + let reread_tip = self.coordinator_client.rc_get_latest_checkpoint_id().await?; + if reread_tip < self.db.get_latest_checkpoint_id().await? { + self.ensure_db_matches_coordinator_head().await?; + } + if reread_tip > target_tip { + target_tip = reread_tip; + continue; + } + self.ensure_db_matches_coordinator_head().await?; + break; + } + Ok(()) + } - self.commit_state( - &coordinator_update, - &prepared_updates, - ProvingJobCircuitType::GUTANoChange, - vec![], - true, + async fn restore_checkpoint_transition( + &mut self, + file_system: &FileSystem, + guta_gatherer_backup_directory: &str, + global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, + database_check_state: DatabaseCheckState, + target_tip: u64, + start: u64, + checkpoint_id: u64, + coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, + pair: psy_data::p2p::RealmTransition, + staged_transitions: &mut HashMap, + rejected_proposal_ids: &mut Vec<[u8; 32]>, + ) -> anyhow::Result { + let target_root = N::QHash::from_owned_32bytes(pair.new_root); + self.state.processing_checkpoint_id = checkpoint_id; + self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + let prepared_updates = if checkpoint_id == 0 { + tracing::info!("Restore target is checkpoint 0 (genesis); using genesis path without backup file."); + self.genesis_recovery_updates(target_root, 0) + } else { + let realm_pending_id = self.db.get_unique_pending_id_for_checkpoint_id(checkpoint_id) + .await?.filter(|_| checkpoint_id >= target_tip); + let Some((realm_unique_pending_id, realm_proc_checkpoint_id)) = realm_pending_id else { + let recovered_from_backup = self.try_pending_backups( + file_system, guta_gatherer_backup_directory, global_user_tree, + database_check_state, checkpoint_id, target_tip, target_root, coordinator_update, ).await?; + if !recovered_from_backup && !self.retry_history_transition( + checkpoint_id, coordinator_update, pair, proposal_store, proposal_fetch, + start, staged_transitions, rejected_proposal_ids, + ).await? { + return Ok(false); + } + self.verify_recovered_root(checkpoint_id, target_root).await?; + return Ok(true); + }; + self.mapped_recovery_updates( + file_system, guta_gatherer_backup_directory, global_user_tree, + checkpoint_id, target_root, realm_unique_pending_id, realm_proc_checkpoint_id, + ).await? + }; + self.commit_state( + coordinator_update, &prepared_updates, ProvingJobCircuitType::GUTANoChange, vec![], + ).await?; + tracing::info!("Checkpoint {} recovered successfully.", checkpoint_id); + self.verify_recovered_root(checkpoint_id, target_root).await?; + Ok(true) + } - tracing::info!("Checkpoint {} recovered successfully.", checkpoint_id); + fn genesis_recovery_updates( + &mut self, + target_root: N::QHash, + proc_checkpoint_unique_id: QCoreProcCheckpointUniqueId, + ) -> PsyPreparedRealmBlockStateUpdates { + self.state.processing_realm_start_root = target_root; + self.state.processing_realm_end_root = target_root; + PsyPreparedRealmBlockStateUpdates { + realm_id: self.state.realm_id_u64, + realm_sub_id: self.state.realm_sub_id_u64, + old_realm_root: target_root, + new_realm_root: target_root, + unique_pending_id: 0, + proc_checkpoint_unique_id, + update_global_user_tree_nodes_ffs: vec![], + update_user_contract_tree_nodes_ffs: vec![], + update_contract_state_tree_nodes_ffs: vec![], + update_user_leaves_ffs: vec![], + update_contract_state_imt_leaves_ffs: vec![], + } + } - let latest_realm_root = self.get_realm_root_from_db().await?; - if latest_realm_root != target_realm_state.value { - anyhow::bail!( - "Post-recovery root mismatch at checkpoint {}! Local: {:?}, Target: {:?}", - checkpoint_id, - latest_realm_root, - target_realm_state.value - ); + async fn mapped_recovery_updates( + &mut self, + file_system: &FileSystem, + guta_gatherer_backup_directory: &str, + global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + checkpoint_id: u64, + target_root: N::QHash, + realm_unique_pending_id: u64, + realm_proc_checkpoint_id: QCoreProcCheckpointUniqueId, + ) -> anyhow::Result> { + if realm_unique_pending_id == 0 { + tracing::info!( + "Restore target checkpoint {} maps to unique_pending_id 0 (no backup file); using genesis-like path.", + checkpoint_id + ); + return Ok(self.genesis_recovery_updates(target_root, realm_proc_checkpoint_id)); + } + self.state.processing_unique_pending_id = realm_unique_pending_id; + self.state.processing_proc_checkpoint_unique_id = realm_proc_checkpoint_id; + self.state.processing_realm_start_root = self.state.last_committed_realm_end_root; + self.state.processing_realm_end_root = target_root; + let prepared = generate_realm_output_from_backups::( + file_system, guta_gatherer_backup_directory, &self.state, + Some(realm_unique_pending_id), global_user_tree, + ).await?; + anyhow::ensure!( + global_user_tree.get_root() == target_root, + "Checkpoint {}: replayed tree root {:?} does not match coordinator target {:?}.", + checkpoint_id, + global_user_tree.get_root(), + target_root + ); + Ok(prepared) + } + + async fn try_pending_backups( + &mut self, + file_system: &FileSystem, + guta_gatherer_backup_directory: &str, + global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + database_check_state: DatabaseCheckState, + checkpoint_id: u64, + target_tip: u64, + target_root: N::QHash, + coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, + ) -> anyhow::Result { + let ((current_unique_pending_id, _), _) = + resolve_current_and_last_committed_pending_ids( + self.state.last_committed_checkpoint_id, + |checkpoint_id| { + let db = self.db.clone(); + async move { db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await } + }, + || self.db.get_latest_mapped_unique_pending_id(), + |unique_pending_id| self.db.get_checkpoint_id_for_unique_pending_id(unique_pending_id), + ).await?; + let last_committed_unique_pending_id = self.state.last_committed_unique_pending_id; + let allow_backup = database_check_state == DatabaseCheckState::NeedsRecovery + && checkpoint_id > self.state.last_committed_checkpoint_id + && checkpoint_id >= target_tip; + if !allow_backup || current_unique_pending_id <= last_committed_unique_pending_id { + return Ok(false); + } + for candidate in (last_committed_unique_pending_id + 1)..=current_unique_pending_id { + let path = get_new_realm_end_cap_gatherer_backup_file_path( + guta_gatherer_backup_directory, self.state.realm_id_u64, + self.state.realm_sub_id_u64, candidate, + ); + match read_realm_backup_end_root::(file_system, &path.to_string_lossy()).await { + Ok(end_root) if end_root == target_root => { + if self.load_matching_backup( + file_system, guta_gatherer_backup_directory, global_user_tree, + checkpoint_id, candidate, target_root, coordinator_update, + ).await? { + return Ok(true); + } } + Ok(end_root) => tracing::debug!( + "Backup pending_id {} end_root {:?} does not match coordinator target {:?} for checkpoint {}.", + candidate, end_root, target_root, checkpoint_id + ), + Err(e) => tracing::debug!( + "Failed to read backup pending_id {} for checkpoint {}: {:?}", + candidate, checkpoint_id, e + ), + } + } + Ok(false) + } - checkpoint_id += 1; + async fn load_matching_backup( + &mut self, + file_system: &FileSystem, + guta_gatherer_backup_directory: &str, + global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + checkpoint_id: u64, + candidate: u64, + target_root: N::QHash, + coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, + ) -> anyhow::Result { + let Some(candidate_proc_checkpoint_id) = + self.db.get_proc_checkpoint_unique_id_for_pending_id(candidate).await? + else { + tracing::warn!( + "Backup pending_id {} matches checkpoint {} end_root but has no durable pending->proc record; skipping instead of borrowing another generation's proc ID.", + candidate, checkpoint_id + ); + return Ok(false); + }; + let mut recovery_state = self.state.clone(); + recovery_state.processing_unique_pending_id = candidate; + recovery_state.processing_proc_checkpoint_unique_id = candidate_proc_checkpoint_id; + recovery_state.processing_realm_start_root = self.state.last_committed_realm_end_root; + recovery_state.processing_realm_end_root = target_root; + tracing::info!( + "Found matching backup for checkpoint {}: pending_id={}. Attempting full load.", + checkpoint_id, candidate + ); + let journal_snapshot = global_user_tree.snapshot(); + match generate_realm_output_from_backups::( + file_system, guta_gatherer_backup_directory, &recovery_state, Some(candidate), global_user_tree, + ).await { + Ok(updates) if updates.new_realm_root == target_root && global_user_tree.get_root() == target_root => { + tracing::info!( + "Backup recovery successful for pending_id {}: end_root matches coordinator target {:?}.", + candidate, target_root + ); + self.state.processing_unique_pending_id = recovery_state.processing_unique_pending_id; + self.state.processing_proc_checkpoint_unique_id = recovery_state.processing_proc_checkpoint_unique_id; + self.state.processing_realm_start_root = recovery_state.processing_realm_start_root; + self.state.processing_realm_end_root = recovery_state.processing_realm_end_root; + self.commit_state(coordinator_update, &updates, ProvingJobCircuitType::GUTANoChange, vec![]).await?; + tracing::info!( + "Checkpoint {} recovered from backup (pending_id={}).", + checkpoint_id, candidate + ); + Ok(true) + } + Ok(updates) => { + global_user_tree.revert_to(journal_snapshot); + tracing::warn!( + "Backup end_root {:?} does not match coordinator target {:?} for pending_id {}. Trying next candidate.", + updates.new_realm_root, target_root, candidate + ); + Ok(false) + } + Err(e) => { + global_user_tree.revert_to(journal_snapshot); + tracing::warn!( + "Backup pending_id {} end_root matches but full load failed: {:?}. Trying next candidate.", + candidate, e + ); + Ok(false) + } + } + } + + async fn retry_history_transition( + &mut self, + checkpoint_id: u64, + coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, + pair: psy_data::p2p::RealmTransition, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, + start: u64, + staged_transitions: &mut HashMap, + rejected_proposal_ids: &mut Vec<[u8; 32]>, + ) -> anyhow::Result { + let included = crate::realm::processor::recovery::CheckpointIdentity { + checkpoint_id, + checkpoint_hash: coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash.into_owned_32bytes(), + }; + for attempt in 0..crate::realm::processor::catchup::CATCHUP_PAIR_ATTEMPTS { + let staged = match staged_transitions.remove(&pair) { + Some(staged) => Some(staged), + None => match proposal_fetch { + Some(client) => self.stage_single_transition( + pair, proposal_store, client, start, rejected_proposal_ids, + ).await?, + None => None, + }, + }; + match self.apply_verified_transition( + &included, pair, staged, proposal_store, rejected_proposal_ids, + ).await { + Ok(Some(_)) => return Ok(true), + Ok(None) => tracing::warn!( + "MissingHistoryProof at C={} attempt={attempt} pair=({},{}) rejected", + checkpoint_id, hex::encode(pair.old_root), hex::encode(pair.new_root) + ), + Err(error) => { + tracing::warn!( + "history apply failed C={checkpoint_id} error={error}; retrying in 5s" + ); + tokio::time::sleep(Duration::from_secs(5)).await; + break; + } } } + // A verification failure can be transient (coordinator material, storage). Drop this + // round's rejections so the next round re-fetches and re-verifies the same candidate. + rejected_proposal_ids.clear(); + tracing::warn!( + "MissingHistoryProof at C={}: no verified candidate for pair=({},{}); retrying in 5s", + checkpoint_id, hex::encode(pair.old_root), hex::encode(pair.new_root) + ); + tokio::time::sleep(Duration::from_secs(5)).await; + Ok(false) + } + async fn verify_recovered_root(&self, checkpoint_id: u64, target_root: N::QHash) -> anyhow::Result<()> { + let latest_realm_root = self.get_realm_root_from_db().await?; + if latest_realm_root != target_root { + anyhow::bail!( + "Post-recovery root mismatch at checkpoint {}! Local: {:?}, Target: {:?}", + checkpoint_id, latest_realm_root, target_root + ); + } Ok(()) } @@ -767,12 +1144,22 @@ where guta_gatherer_backup_directory: &str, genesis_block_update: PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, + proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, ) -> anyhow::Result<()> { let genesis_checkpoint_root = genesis_block_update.coordinator_update.checkpoint_sync_info.checkpoint_tree_root; self.ensure_genesis_applied(genesis_block_update).await?; - - self.ensure_backup_restored_if_necessary(file_system, guta_gatherer_backup_directory, global_user_tree) + let genesis_tip = self.db.get_latest_checkpoint_id().await?; + self.publish_validator_leaves(proposal_fetch, genesis_tip).await?; + + self.ensure_backup_restored_if_necessary( + file_system, + guta_gatherer_backup_directory, + global_user_tree, + proposal_store, + proposal_fetch, + ) .await?; if self.state.last_committed_checkpoint_id > 0 { @@ -781,13 +1168,7 @@ where .await?; } - let current_realm_root = self.db.global_user_tree_get_node(self.state.last_committed_checkpoint_id, self.realm_root_node).await?; - - self.state.last_committed_realm_end_root = current_realm_root; - self.state.last_committed_realm_start_root = current_realm_root; - self.state.processing_realm_start_root = current_realm_root; - self.state.processing_realm_end_root = current_realm_root; - self.state.gathering_realm_start_root = current_realm_root; + self.set_committed_realm_roots_from_db().await?; self.sync_to_coordinator_set_checkpoint_id().await?; @@ -812,7 +1193,7 @@ where }; self.state.last_committed_checkpoint_root = last_committed_checkpoint_root; - self.set_new_unique_ids(Some(current_realm_root)).await?; + self.set_new_unique_ids(Some(self.state.last_committed_realm_end_root)).await?; self.guta_queue_key_status_manager .set_unique_id(self.state.gathering_proc_checkpoint_unique_id)?; @@ -823,7 +1204,7 @@ where "[REALM] Initialized. Checkpoint: {}, Pending ID: {}, Realm Root: {:?}", self.state.coordinator_head_synced_checkpoint_id, self.state.gathering_unique_pending_id, - current_realm_root + self.state.last_committed_realm_end_root ); self.print_coordinator_processor_state(); Ok(()) @@ -832,7 +1213,12 @@ where #[cfg(test)] mod tests { - use super::{ensure_latest_pending_within_target, find_latest_mapped_pending_at_or_before, resolve_current_and_last_committed_pending_ids}; + use super::{ + ensure_latest_pending_within_target, find_latest_mapped_pending_at_or_before, + resolve_current_and_last_committed_pending_ids, + }; + use crate::realm::processor::db::genesis::classify_genesis_mapping; + use crate::realm::processor::db::DatabaseCheckState; use std::collections::HashMap; use std::future::Future; @@ -851,6 +1237,25 @@ mod tests { } } + #[test] + fn applied_genesis_is_not_needs_genesis() { + assert_eq!( + classify_genesis_mapping(0, Ok(None)).expect("empty store"), + Some(DatabaseCheckState::NeedsGenesis) + ); + assert_eq!( + classify_genesis_mapping(0, Ok(Some((0, 0)))).expect("applied genesis"), + None + ); + assert_eq!( + classify_genesis_mapping(1, Ok(None)).expect("later checkpoint"), + None + ); + let lookup_error = classify_genesis_mapping(0, Err(anyhow::anyhow!("mapping unavailable"))) + .expect_err("lookup errors must propagate"); + assert!(lookup_error.to_string().contains("mapping unavailable"), "{lookup_error}"); + } + #[tokio::test] async fn missing_mapping_walks_back_to_latest_mapping() { let mappings = HashMap::from([(197u64, (87u64, 10087u128))]); diff --git a/psy_node_common/src/realm/processor/db/mod.rs b/psy_node_common/src/realm/processor/db/mod.rs index b52b2517..23504b3c 100644 --- a/psy_node_common/src/realm/processor/db/mod.rs +++ b/psy_node_common/src/realm/processor/db/mod.rs @@ -1,6 +1,7 @@ mod core; pub use core::*; mod commit; +mod genesis; mod init; mod sync; mod sanity_check; \ No newline at end of file diff --git a/psy_node_common/src/realm/processor/db/sync.rs b/psy_node_common/src/realm/processor/db/sync.rs index 1b211ab2..3ac0334c 100644 --- a/psy_node_common/src/realm/processor/db/sync.rs +++ b/psy_node_common/src/realm/processor/db/sync.rs @@ -1,7 +1,16 @@ -use anyhow::Ok; +use anyhow::{Context, Ok}; use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; -use parth_core::protocol::core_types::QNetworkTypesConfig; -use psy_data::{prepared_block::realm::PsyRealmCoordinatorUpdate, v1::qdata::checkpoint::QEDL2BlockState}; +use parth_core::{ + crypto::hash::{ + merkle_proof::MerkleProofCore, + traits::{FieldQHasher, MerkleZeroHasher, ZeroableHash}, + }, + protocol::core_types::QNetworkTypesConfig, +}; +use psy_data::{ + prepared_block::realm::PsyRealmCoordinatorUpdate, + v1::qdata::{checkpoint::QEDL2BlockState, checkpoint_sync::PQEDCheckpointSyncInfoCompact}, +}; use psy_io::tokio::TokioLikeFileSystem; use psy_node_core::{ p2p::traits::realm_coordinantor::RealmCoordinatorClient, @@ -15,6 +24,70 @@ use psy_node_core::{ use crate::realm::processor::db::PsyRealmDatabaseProcessor; +fn require_checkpoint_metadata( + checkpoint_id: u64, + realm_id: u64, + checkpoint_tree_height: usize, + coordinator_global_user_tree_height: usize, + checkpoint_sync: &PQEDCheckpointSyncInfoCompact, + membership: &MerkleProofCore, + local_proof: &MerkleProofCore, + previous_root: Hash, + realm_proof: &MerkleProofCore, +) -> anyhow::Result<()> +where + F: parth_core::felt::QFelt64, + Hash: parth_core::protocol::core_types::QFHashBase, + H: FieldQHasher + MerkleZeroHasher, +{ + anyhow::ensure!( + checkpoint_sync.checkpoint_id == checkpoint_id + && checkpoint_sync.block_state.checkpoint_id == checkpoint_id, + "MissingHistoryProof at C={checkpoint_id}: coordinator metadata checkpoint IDs do not match C" + ); + anyhow::ensure!( + membership.index == checkpoint_id, + "MissingHistoryProof at C={checkpoint_id}: membership index {} is not C", + membership.index + ); + anyhow::ensure!( + membership.siblings.len() == checkpoint_tree_height && membership.siblings.len() <= 64, + "MissingHistoryProof at C={checkpoint_id}: membership height {} is not {}", + membership.siblings.len(), + checkpoint_tree_height + ); + anyhow::ensure!( + membership.value == checkpoint_sync.checkpoint_leaf_hash && membership.value == local_proof.value, + "MissingHistoryProof at C={checkpoint_id}: membership value does not match checkpoint leaf hash" + ); + anyhow::ensure!( + membership.root == checkpoint_sync.checkpoint_tree_root, + "MissingHistoryProof at C={checkpoint_id}: membership root does not match synchronized C root" + ); + checkpoint_sync.ensure_valid::(&membership.siblings)?; + anyhow::ensure!( + membership.compute_root_with_value::(Hash::get_zero_value()) == previous_root, + "MissingHistoryProof at C={checkpoint_id}: empty-leaf root does not match synchronized C-1 root" + ); + anyhow::ensure!( + realm_proof.verify::(), + "MissingHistoryProof at C={checkpoint_id}: Realm top proof does not verify" + ); + anyhow::ensure!( + realm_proof.index == realm_id, + "MissingHistoryProof at C={checkpoint_id}: Realm top proof index mismatch" + ); + anyhow::ensure!( + realm_proof.siblings.len() == coordinator_global_user_tree_height, + "MissingHistoryProof at C={checkpoint_id}: Realm top proof height mismatch" + ); + anyhow::ensure!( + realm_proof.root == checkpoint_sync.state_roots.user_tree_root, + "MissingHistoryProof at C={checkpoint_id}: Realm top proof is not bound to C user tree root" + ); + Ok(()) +} + impl< N: QNetworkTypesConfig, S: PsyRealmProcessorStore + Send + Sync, @@ -326,7 +399,7 @@ where } } - async fn persist_checkpoint_metadata_range( + pub(super) async fn persist_checkpoint_metadata_range( &mut self, from_checkpoint_id: u64, to_checkpoint_id: u64, @@ -342,79 +415,72 @@ where .coordinator_client .rc_get_realm_sync_info(checkpoint_id, self.state.realm_id_u64) .await?; - - // CRITICAL VALIDATION: Ensure the local in-memory tree matches the Coordinator's canonical root for this checkpoint. - // If we have diverged (e.g. bad leaves or fork), we must reset the Backup Manager. - // We retrieve the proof for the leaf at `checkpoint_id`. The `get_append_root` from that proof - // represents the root of the tree at the moment that leaf was the right-most element (i.e., at that checkpoint). + let membership = self + .coordinator_client + .rc_get_checkpoint_tree_merkle_proof(checkpoint_id) + .await + .with_context(|| { + format!("MissingHistoryProof at C={checkpoint_id}: checkpoint tree membership unavailable") + })?; + let checkpoint_sync = &sync_info.checkpoint_sync_info; let local_proof = self.checkpoint_tree_backup_manager.checkpoint_tree.get_leaf(checkpoint_id); - let local_calculated_root = local_proof.get_append_root::(); - - if local_calculated_root != sync_info.checkpoint_sync_info.checkpoint_tree_root { - tracing::error!( - "CRITICAL CHECKSUM MISMATCH: Local Checkpoint Tree Root {:?} != Coordinator Root {:?} at Checkpoint {}. Triggering Backup Manager Hard Reset.", - local_calculated_root, - sync_info.checkpoint_sync_info.checkpoint_tree_root, - checkpoint_id - ); - - // Reset the backup manager to the last known committed state in the DB to clear invalid in-memory state. + let local_root = local_proof.get_append_root::(); + if local_root != checkpoint_sync.checkpoint_tree_root { self.checkpoint_tree_backup_manager .hard_reset_and_truncate(reset_checkpoint_id) .await?; - - anyhow::bail!("Checkpoint Tree Divergence detected at checkpoint {}. Local state reset. Please retry sync.", checkpoint_id); + anyhow::bail!( + "Checkpoint Tree Divergence detected at checkpoint {checkpoint_id}. Local state reset. Please retry sync." + ); } + let previous_root = if checkpoint_id == 0 { + N::HasherBase::get_zero_hash(N::CHECKPOINT_TREE_HEIGHT as usize) + } else { + self.checkpoint_tree_backup_manager + .checkpoint_tree + .get_leaf(checkpoint_id - 1) + .get_append_root::() + }; + let realm_proof = &sync_info.merkle_proof_to_realm_root; + require_checkpoint_metadata::( + checkpoint_id, + self.state.realm_id_u64, + N::CHECKPOINT_TREE_HEIGHT as usize, + N::COORDINATOR_GLOBAL_USER_TREE_HEIGHT as usize, + checkpoint_sync, + &membership, + &local_proof, + previous_root, + realm_proof, + )?; tracing::info!( "sync checkpoint metadata: checkpoint_id={}, checkpoint_tree_root={:?}, block_state_checkpoint_id={}", checkpoint_id, - sync_info.checkpoint_sync_info.checkpoint_tree_root, - sync_info.checkpoint_sync_info.block_state.checkpoint_id - ); - // ORDERING IS LOAD-BEARING: these writes are not transactional, so a crash between them can leave a - // checkpoint half-written. Recovery (`try_get_complete_l2_block_state`) requires all dependency records, - // and the L2 block state is written LAST so that its presence implies every other record was already - // written. Writing it earlier would let a crash after the block state but before the roots/leaf/proofs - // leave a checkpoint that recovery believes is complete and never re-syncs. - tracing::debug!( - "set checkpoint global state roots {} {:?}", - checkpoint_id, - sync_info.checkpoint_sync_info.state_roots + checkpoint_sync.checkpoint_tree_root, + checkpoint_sync.block_state.checkpoint_id ); self.db - .set_checkpoint_global_state_roots(checkpoint_id, &sync_info.checkpoint_sync_info.state_roots) + .set_checkpoint_global_state_roots(checkpoint_id, &checkpoint_sync.state_roots) .await?; self.db - .set_checkpoint_leaf_data(checkpoint_id, &sync_info.checkpoint_sync_info.checkpoint_leaf) + .set_checkpoint_leaf_data(checkpoint_id, &checkpoint_sync.checkpoint_leaf) .await?; - tracing::debug!( - "committing checkpoint proof: {:?}", - &local_proof.to_append_proof::() - ); - self.db - .checkpoint_tree_injest_merkle_proof(checkpoint_id, &local_proof.to_append_proof::()) + .checkpoint_tree_injest_merkle_proof(checkpoint_id, &membership) .await?; self.db - .set_checkpoint_root_hash_to_id_mapping( - sync_info.checkpoint_sync_info.checkpoint_tree_root, - sync_info.checkpoint_sync_info.checkpoint_id, - ) + .set_checkpoint_root_hash_to_id_mapping(checkpoint_sync.checkpoint_tree_root, checkpoint_id) .await?; - self.db - .global_user_tree_set_top_tree_merkle_proof(checkpoint_id, &sync_info.merkle_proof_to_realm_root) + .global_user_tree_set_top_tree_merkle_proof(checkpoint_id, realm_proof) .await?; - - // Sentinel write — must remain the final persisted metadata for this checkpoint (see note above). self.db - .set_l2_block_state(checkpoint_id, &sync_info.checkpoint_sync_info.block_state) + .set_l2_block_state(checkpoint_id, &checkpoint_sync.block_state) .await?; latest_sync_info = Some(sync_info); } - Ok(latest_sync_info) } @@ -481,3 +547,192 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use super::require_checkpoint_metadata; + use parth_common::memory_stores::dash_tree_append_only::PsyDashMemoryAppendOnlyMerkleStore; + use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; + use parth_core::crypto::hash::merkle_proof::MerkleProofCore; + use parth_core::crypto::hash::traits::{FieldQHasher, FromU64x4, MerkleZeroHasher, QFieldHashable, ZeroableHash}; + use parth_core::pgoldilocks::PoseidonHasher; + use parth_core::{PF, PHash}; + use psy_data::v1::qdata::checkpoint::{ + PQEDCheckpointGlobalStateRoots, PQEDCheckpointLeaf, PQEDCheckpointLeafStats, QEDL2BlockState, + }; + use psy_data::v1::qdata::checkpoint_sync::PQEDCheckpointSyncInfoCompact; + + const CHECKPOINT_HEIGHT: usize = 8; + const REALM_HEIGHT: usize = 4; + const REALM_ID: u64 = 0; + + struct Fixture { + checkpoint_id: u64, + previous_root: PHash, + local_proof: MerkleProofCore, + checkpoint_sync: PQEDCheckpointSyncInfoCompact, + membership: MerkleProofCore, + realm_proof: MerkleProofCore, + } + + fn zero_siblings(height: usize) -> Vec { + (0..height).map(|level| PoseidonHasher::get_zero_hash(level)).collect() + } + + fn empty_roots(user_tree_root: PHash) -> PQEDCheckpointGlobalStateRoots { + PQEDCheckpointGlobalStateRoots { + contract_tree_root: PHash::get_zero_value(), + deposit_tree_root: PHash::get_zero_value(), + user_tree_root, + withdrawal_tree_root: PHash::get_zero_value(), + user_registration_tree_root: PHash::get_zero_value(), + validator_tree_root: PHash::get_zero_value(), + } + } + + fn genesis_fixture() -> Fixture { + let realm_value = PHash::from_u64x4([7, 0, 0, 0]); + let realm_proof = MerkleProofCore::new_from_params::(REALM_ID, realm_value, zero_siblings(REALM_HEIGHT)); + let state_roots = empty_roots(realm_proof.root); + let leaf = PQEDCheckpointLeaf { + global_chain_root: state_roots.qfhash::(), + stats: PQEDCheckpointLeafStats::get_empty_stats(), + }; + let leaf_hash = leaf.qfhash::(); + let tree = PsyDashMemoryAppendOnlyMerkleStore::::new(CHECKPOINT_HEIGHT as u8); + tree.append_leaf(0, leaf_hash).unwrap(); + let local_proof = tree.get_leaf(0); + let membership = MerkleProofCore::new_from_params::(0, leaf_hash, zero_siblings(CHECKPOINT_HEIGHT)); + let checkpoint_sync = PQEDCheckpointSyncInfoCompact { + checkpoint_id: 0, + coordinator_id: 0, + coordinator_sub_id: 0, + coordinator_unique_pending_id: 0, + block_state: QEDL2BlockState::get_genesis_value(), + state_roots, + checkpoint_leaf: leaf, + checkpoint_leaf_hash: leaf_hash, + checkpoint_tree_root: local_proof.get_append_root::(), + }; + Fixture { + checkpoint_id: 0, + previous_root: PoseidonHasher::get_zero_hash(CHECKPOINT_HEIGHT), + local_proof, + checkpoint_sync, + membership, + realm_proof, + } + } + + fn historical_fixture() -> Fixture { + let realm_value = PHash::from_u64x4([11, 0, 0, 0]); + let realm_proof = MerkleProofCore::new_from_params::(REALM_ID, realm_value, zero_siblings(REALM_HEIGHT)); + let first_roots = empty_roots(PHash::from_u64x4([1, 0, 0, 0])); + let first_leaf = PQEDCheckpointLeaf { + global_chain_root: first_roots.qfhash::(), + stats: PQEDCheckpointLeafStats::get_empty_stats(), + }; + let first_hash = first_leaf.qfhash::(); + let second_roots = empty_roots(realm_proof.root); + let second_leaf = PQEDCheckpointLeaf { + global_chain_root: second_roots.qfhash::(), + stats: PQEDCheckpointLeafStats::get_empty_stats(), + }; + let second_hash = second_leaf.qfhash::(); + let tree = PsyDashMemoryAppendOnlyMerkleStore::::new(CHECKPOINT_HEIGHT as u8); + tree.append_leaf(0, first_hash).unwrap(); + let previous_root = tree.get_leaf(0).get_append_root::(); + tree.append_leaf(1, second_hash).unwrap(); + let local_proof = tree.get_leaf(1); + let membership = tree.get_historical_merkle_proof_at_historical_index(1, 1); + let checkpoint_sync = PQEDCheckpointSyncInfoCompact { + checkpoint_id: 1, + coordinator_id: 0, + coordinator_sub_id: 0, + coordinator_unique_pending_id: 0, + block_state: QEDL2BlockState { + checkpoint_id: 1, + ..QEDL2BlockState::get_genesis_value() + }, + state_roots: second_roots, + checkpoint_leaf: second_leaf, + checkpoint_leaf_hash: second_hash, + checkpoint_tree_root: local_proof.get_append_root::(), + }; + Fixture { + checkpoint_id: 1, + previous_root, + local_proof, + checkpoint_sync, + membership, + realm_proof, + } + } + + fn require_fixture(fixture: &Fixture) -> anyhow::Result<()> { + require_checkpoint_metadata::( + fixture.checkpoint_id, + REALM_ID, + CHECKPOINT_HEIGHT, + REALM_HEIGHT, + &fixture.checkpoint_sync, + &fixture.membership, + &fixture.local_proof, + fixture.previous_root, + &fixture.realm_proof, + ) + } + + #[test] + fn require_checkpoint_metadata_accepts_genesis_and_historical_proofs() { + require_fixture(&genesis_fixture()).expect("genesis metadata must authenticate"); + require_fixture(&historical_fixture()).expect("historical metadata must authenticate"); + } + + #[test] + fn require_checkpoint_metadata_rejects_altered_bindings() { + let mut fixture = genesis_fixture(); + fixture.checkpoint_sync.state_roots.contract_tree_root = PHash::from_u64x4([9, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.checkpoint_sync.checkpoint_leaf_hash = PHash::from_u64x4([3, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.membership.index = 1; + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.membership.value = PHash::from_u64x4([4, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.membership.root = PHash::from_u64x4([5, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.membership.siblings.push(PHash::get_zero_value()); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.previous_root = PHash::from_u64x4([6, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.local_proof.value = PHash::from_u64x4([8, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.realm_proof.index = 1; + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.realm_proof.root = PHash::from_u64x4([12, 0, 0, 0]); + assert!(require_fixture(&fixture).is_err()); + + let mut fixture = genesis_fixture(); + fixture.realm_proof.siblings.truncate(REALM_HEIGHT - 1); + assert!(require_fixture(&fixture).is_err()); + } +} diff --git a/psy_node_common/src/realm/processor/gatherers/realm_end_cap_gatherer.rs b/psy_node_common/src/realm/processor/gatherers/realm_end_cap_gatherer.rs index fcc66e10..86443179 100644 --- a/psy_node_common/src/realm/processor/gatherers/realm_end_cap_gatherer.rs +++ b/psy_node_common/src/realm/processor/gatherers/realm_end_cap_gatherer.rs @@ -1699,3 +1699,33 @@ mod backup_file_tests { } } } + +#[cfg(test)] +mod history_overlap_tests { + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; + use parth_core::{ + pgoldilocks::PoseidonHasher, + protocol::core_types::Q256BitHash, + PHash, + }; + + use super::apply_state_updates_to_tree; + + #[test] + fn history_live_overlap_regression() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(4); + let start = tree.get_root(); + apply_state_updates_to_tree(&mut tree, start, start, &[], 32, 0).unwrap(); + assert_eq!(tree.get_last_commit_root(), start); + apply_state_updates_to_tree(&mut tree, start, start, &[], 32, 0).unwrap(); + assert_eq!(tree.get_last_commit_root(), start); + let other = PHash::from_owned_32bytes([0xAB; 32]); + let error = apply_state_updates_to_tree(&mut tree, start, other, &[], 32, 0).unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("new_root"), + "{message}" + ); + assert_eq!(tree.get_last_commit_root(), start); + } +} diff --git a/psy_node_common/src/realm/processor/mod.rs b/psy_node_common/src/realm/processor/mod.rs index 83e171df..f8d95acf 100644 --- a/psy_node_common/src/realm/processor/mod.rs +++ b/psy_node_common/src/realm/processor/mod.rs @@ -1,7 +1,10 @@ +pub mod catchup; pub mod consensus; pub mod core; pub mod db; pub mod processor_shared_status; pub mod gatherers; pub mod create; -pub mod coordinator_sync; \ No newline at end of file +pub mod coordinator_sync; +pub mod proposal_store; +pub mod recovery; \ No newline at end of file diff --git a/psy_node_common/src/realm/processor/proposal_store.rs b/psy_node_common/src/realm/processor/proposal_store.rs new file mode 100644 index 00000000..2f31924f --- /dev/null +++ b/psy_node_common/src/realm/processor/proposal_store.rs @@ -0,0 +1,509 @@ +//! Disk-backed Proposal objects. + +use std::collections::HashMap; +use std::hash::{BuildHasher, Hasher}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context; +use psy_data::p2p::{ + BodyChunkRequest, BodyChunkResponse, Proposal, ProposalLookupEntry, ProposalLookupRequest, + ProposalLookupResponse, ProtocolEncode, BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, + PROPOSAL_WIRE_BYTES, +}; +use psy_io::tokio::{TokioFileLike, TokioLikeFileSystem, TokioStdFileSystem}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +use crate::realm::processor::consensus::decode_proposal_body; + +const STATE_UPDATES_ROOTS_OFFSET: usize = 40; +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct RealmTransition { + pub from_root: [u8; 32], + pub to_root: [u8; 32], +} + +struct TransitionRecord { + body_file: T, + body_len_bytes: u64, + body_hash: [u8; 32], + proposal: Proposal, +} + +/// Owns one staged body file. Dropping it removes the staging file through the +/// store's filesystem abstraction, so an abandoned candidate cannot leak. +pub struct StagedProposal { + path: String, + transition: RealmTransition, + proposal_id: [u8; 32], + cleanup: Option>, +} + +impl Drop for StagedProposal { + fn drop(&mut self) { + if let Some(cleanup) = self.cleanup.take() { + cleanup(); + } + } +} + +/// In-memory tables over the retained bodies, guarded by the store's single +/// lock: records keyed by transition plus the O(1) proposal-id lookup index. +struct RetainedBodies { + by_transition: HashMap>, + by_proposal_id: HashMap<[u8; 32], RealmTransition>, +} + +pub struct ProposalStore { + fs: std::sync::Arc, + root: PathBuf, + inner: tokio::sync::Mutex>, + staged_seq: AtomicU64, + instance_id: u64, +} + +impl ProposalStore { + pub async fn open(root: impl AsRef) -> anyhow::Result { + Self::open_with_fs(root, TokioStdFileSystem).await + } +} + +impl ProposalStore { + pub async fn open_with_fs(root: impl AsRef, fs: F) -> anyhow::Result { + let root = root.as_ref().to_path_buf(); + fs.file_like_fs_create_dir_all(&path_string(&root.join("bodies"))).await?; + let root = if root.is_absolute() { root } else { std::env::current_dir()?.join(root) }; + let store = Self { + fs: std::sync::Arc::new(fs), + root, + inner: tokio::sync::Mutex::new(RetainedBodies { + by_transition: HashMap::new(), + by_proposal_id: HashMap::new(), + }), + staged_seq: AtomicU64::new(1), + instance_id: std::hash::RandomState::new().build_hasher().finish(), + }; + remove_staged_files(&store.root.join("bodies")).await?; + { + let mut inner = store.inner.lock().await; + for transition in store.read_stored_transitions().await? { + let _ = store.load_transition_record(&mut inner, &transition).await; + } + } + Ok(store) + } + + // Trusted local consensus/replay writes bypass fetched-body isolation staging verification. + pub async fn save_proposal(&self, proposal: &Proposal, body: &[u8]) -> anyhow::Result<()> { + let staged = self.create_staged(proposal, body).await?; + self.install(staged).await + } + + pub async fn create_staged(&self, proposal: &Proposal, body: &[u8]) -> anyhow::Result { + let transition = verify_complete_object(proposal, body)?; + let path = self.write_staging_file(&encode_object(proposal, body)).await?; + let cleanup_fs = self.fs.clone(); + let cleanup_path = path.clone(); + let handle = tokio::runtime::Handle::current(); + Ok(StagedProposal { + path, + transition, + proposal_id: proposal.proposal_id, + cleanup: Some(Box::new(move || { + let _ = handle.spawn(async move { + if let Err(error) = cleanup_fs.file_like_remove_file(&cleanup_path).await { + if error.kind() != std::io::ErrorKind::NotFound { + tracing::debug!("staged body cleanup failed error={error}"); + } + } + }); + })), + }) + } + + pub async fn read_staged(&self, staged: &StagedProposal) -> anyhow::Result<(Proposal, Vec)> { + let bytes = self.read_object_file(&staged.path).await?; + let object = decode_verified_object(&bytes, &staged.transition)?; + anyhow::ensure!(object.0.proposal_id == staged.proposal_id, "staged identity mismatch"); + Ok(object) + } + + pub async fn lookup_proposal(&self, request: &ProposalLookupRequest) -> anyhow::Result { + let mut entries = Vec::new(); + let mut wire_bytes = 5; + for pair in &request.pairs { + let candidates = self.lookup_transition(&pair.old_root, &pair.new_root).await? + .into_iter().filter(|proposal| proposal.chain_id == request.chain_id && proposal.realm_id == request.realm_id).collect::>(); + wire_bytes += 65 + candidates.len() * PROPOSAL_WIRE_BYTES; + if wire_bytes > psy_data::p2p::MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { + return Ok(ProposalLookupResponse::truncated(entries)); + } + entries.push(ProposalLookupEntry { transition: *pair, candidates }); + } + Ok(ProposalLookupResponse::candidates(entries)) + } + + /// The store retains one body per transition, so this yields zero or one + /// proposal even though the wire answer carries room for two candidates. + /// Cache hits skip disk reads; later on-disk corruption surfaces only on + /// the verified load paths (load_proposal / read_body_chunk I/O). + pub async fn lookup_transition(&self, old_root: &[u8; 32], new_root: &[u8; 32]) -> anyhow::Result> { + { + let inner = self.inner.lock().await; + if let Some(record) = inner + .by_transition + .get(&RealmTransition { from_root: *old_root, to_root: *new_root }) + { + return Ok(vec![record.proposal.clone()]); + } + } + match self.load_proposal(old_root, new_root).await { + Ok(Some((proposal, _))) => Ok(vec![proposal]), + Ok(None) => Ok(Vec::new()), + Err(_) => Ok(Vec::new()), + } + } + + pub async fn load_proposal(&self, old_root: &[u8; 32], new_root: &[u8; 32]) -> anyhow::Result)>> { + let mut inner = self.inner.lock().await; + match self + .load_transition_record(&mut inner, &RealmTransition { from_root: *old_root, to_root: *new_root }) + .await + { + Ok(object) => Ok(Some(object)), + Err(error) if error.downcast_ref::().is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => Ok(None), + Err(error) => Err(error), + } + } + + pub async fn read_body_chunk(&self, request: &BodyChunkRequest) -> anyhow::Result { + anyhow::ensure!(request.max_bytes > 0 && request.max_bytes <= BODY_CHUNK_MAX_BYTES, "range size outside bounds"); + let mut inner = self.inner.lock().await; + let transition = *inner + .by_proposal_id + .get(&request.proposal_id) + .context("proposal is missing")?; + let state = inner + .by_transition + .get_mut(&transition) + .context("proposal is missing")?; + let (body_len, body_hash) = (state.body_len_bytes, state.body_hash); + let file = &mut state.body_file; + anyhow::ensure!(request.offset <= body_len, "range offset past body length"); + let take = (body_len - request.offset).min(request.max_bytes as u64) as usize; + let start = (PROPOSAL_WIRE_BYTES as u64).checked_add(request.offset).context("range seek overflow")?; + let mut data = vec![0; take]; + let result = async { + file.seek(std::io::SeekFrom::Start(start)).await?; + file.read_exact(&mut data).await?; + Ok::<_, std::io::Error>(()) + }.await; + if let Err(error) = result { + self.remove_transition(&mut inner, &transition).await; + return Err(error.into()); + } + Ok(BodyChunkResponse { offset: request.offset, eof: request.offset + take as u64 == body_len, data, body_len, body_hash }) + } + + async fn load_transition_record( + &self, + inner: &mut RetainedBodies, + transition: &RealmTransition, + ) -> anyhow::Result<(Proposal, Vec)> { + let result = async { + let mut file = self.fs.file_like_fs_open(&self.transition_path(transition)).await?; + let mut bytes = Vec::new(); + (&mut file).take((PROPOSAL_WIRE_BYTES + MAX_PROPOSAL_BODY_BYTES + 1) as u64).read_to_end(&mut bytes).await?; + let (proposal, body) = decode_verified_object(&bytes, transition)?; + if let Some(state) = inner.by_transition.get(transition) { + anyhow::ensure!( + state.proposal.proposal_id == proposal.proposal_id, + "proposal identity mismatch" + ); + } + let proposal_id = proposal.proposal_id; + inner.by_proposal_id.insert(proposal_id, *transition); + inner.by_transition.insert(*transition, TransitionRecord { + body_file: file, + body_len_bytes: body.len() as u64, + body_hash: proposal.body_hash, + proposal: proposal.clone(), + }); + Ok((proposal, body)) + }.await; + if result.is_err() { + self.remove_transition(inner, transition).await; + } + result + } + + /// A transition whose bytes do not verify is dropped entirely: the pair goes + /// back to absent so an honest body can be staged and installed again. + async fn remove_transition(&self, inner: &mut RetainedBodies, transition: &RealmTransition) { + if let Some(record) = inner.by_transition.remove(transition) { + inner.by_proposal_id.remove(&record.proposal.proposal_id); + } + let path = self.transition_path(transition); + if let Err(error) = self.fs.file_like_remove_file(&path).await { + if error.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + "failed to remove unverified transition pair=({},{}) error={error}", + hex::encode(transition.from_root), + hex::encode(transition.to_root) + ); + } + } + } + + /// Verifies the staged bytes and installs them as the pair's retained body. + pub async fn install(&self, staged: StagedProposal) -> anyhow::Result<()> { + let mut inner = self.inner.lock().await; + let mut file = self.fs.file_like_fs_open(&staged.path).await?; + let mut bytes = Vec::new(); + (&mut file).take((PROPOSAL_WIRE_BYTES + MAX_PROPOSAL_BODY_BYTES + 1) as u64).read_to_end(&mut bytes).await?; + anyhow::ensure!(bytes.len() >= PROPOSAL_WIRE_BYTES, "short proposal header"); + let proposal = Proposal::decode_exact(&bytes[..PROPOSAL_WIRE_BYTES]).map_err(|error| anyhow::anyhow!("proposal header: {error}"))?; + let body = &bytes[PROPOSAL_WIRE_BYTES..]; + anyhow::ensure!( + verify_complete_object(&proposal, body)? == staged.transition, + "proposal transition mismatch" + ); + anyhow::ensure!(proposal.proposal_id == staged.proposal_id, "staged identity mismatch"); + let path = self.transition_path(&staged.transition); + self.fs.file_like_rename(&staged.path, &path).await?; + let proposal_id = proposal.proposal_id; + let previous = inner.by_transition.insert(staged.transition, TransitionRecord { + body_file: file, + body_len_bytes: body.len() as u64, + body_hash: proposal.body_hash, + proposal, + }); + if let Some(previous) = previous { + if previous.proposal.proposal_id != proposal_id { + inner.by_proposal_id.remove(&previous.proposal.proposal_id); + } + } + inner.by_proposal_id.insert(proposal_id, staged.transition); + self.fs.file_like_fs_sync_parent_dir(&path).await?; + Ok(()) + } + + async fn read_stored_transitions(&self) -> anyhow::Result> { + let mut transitions = Vec::new(); + let mut dir = tokio::fs::read_dir(self.root.join("bodies")).await?; + while let Some(entry) = dir.next_entry().await? { + if let Some(transition) = parse_transition_file_name(&entry.file_name().to_string_lossy()) { + transitions.push(transition); + } + } + transitions.sort_unstable(); + Ok(transitions) + } + + async fn write_staging_file(&self, bytes: &[u8]) -> anyhow::Result { + let tmp = path_string(&self.root.join("bodies").join(format!( + ".tmp-{}-{}", + self.instance_id, + self.staged_seq.fetch_add(1, Ordering::Relaxed) + ))); + let result = async { + let mut file = self.fs.file_like_fs_create(&tmp).await?; + file.write_all(bytes).await?; + file.file_like_set_len(bytes.len() as u64).await?; + self.fs.file_like_fs_flush_file_with_path(&tmp, &mut file).await?; + self.fs.file_like_fs_sync_file_with_path(&tmp, &mut file).await?; + Ok::<_, anyhow::Error>(()) + }.await; + if result.is_err() { + let _ = self.fs.file_like_remove_file(&tmp).await; + } + result.map(|_| tmp) + } + + async fn read_object_file(&self, path: &str) -> anyhow::Result> { + let file = self.fs.file_like_fs_open(path).await?; + let mut bytes = Vec::new(); + file.take((PROPOSAL_WIRE_BYTES + MAX_PROPOSAL_BODY_BYTES + 1) as u64).read_to_end(&mut bytes).await?; + Ok(bytes) + } + + fn transition_path(&self, transition: &RealmTransition) -> String { + path_string(&self.root.join("bodies").join(format!( + "{}_{}", + hex::encode(transition.from_root), + hex::encode(transition.to_root) + ))) + } +} + +fn path_string(path: &Path) -> String { path.to_string_lossy().into_owned() } + +fn encode_object(proposal: &Proposal, body: &[u8]) -> Vec { + let mut bytes = proposal.protocol_encode_to_vec(); + bytes.extend_from_slice(body); + bytes +} + +fn decode_verified_object(bytes: &[u8], transition: &RealmTransition) -> anyhow::Result<(Proposal, Vec)> { + anyhow::ensure!(bytes.len() >= PROPOSAL_WIRE_BYTES, "short proposal header"); + let proposal = Proposal::decode_exact(&bytes[..PROPOSAL_WIRE_BYTES]).map_err(|error| anyhow::anyhow!("proposal header: {error}"))?; + let body = &bytes[PROPOSAL_WIRE_BYTES..]; + anyhow::ensure!( + verify_complete_object(&proposal, body)? == *transition, + "proposal transition mismatch" + ); + Ok((proposal, body.to_vec())) +} + +fn verify_complete_object(proposal: &Proposal, body: &[u8]) -> anyhow::Result { + anyhow::ensure!(body.len() <= MAX_PROPOSAL_BODY_BYTES, "proposal body exceeds length limit"); + anyhow::ensure!(proposal.compute_proposal_id() == proposal.proposal_id, "proposal identity mismatch"); + let decoded = decode_proposal_body(proposal, body).map_err(|error| anyhow::anyhow!("proposal body: {error}"))?; + realm_roots_from_state_updates(&decoded.state_updates) +} + +fn realm_roots_from_state_updates(state_updates: &[u8]) -> anyhow::Result { + anyhow::ensure!(state_updates.len() >= STATE_UPDATES_ROOTS_OFFSET + 64, "state_updates missing old/new realm roots"); + let mut old_root = [0; 32]; + let mut new_root = [0; 32]; + old_root.copy_from_slice(&state_updates[STATE_UPDATES_ROOTS_OFFSET..STATE_UPDATES_ROOTS_OFFSET + 32]); + new_root.copy_from_slice(&state_updates[STATE_UPDATES_ROOTS_OFFSET + 32..STATE_UPDATES_ROOTS_OFFSET + 64]); + Ok(RealmTransition { from_root: old_root, to_root: new_root }) +} + +async fn remove_staged_files(dir: &Path) -> anyhow::Result<()> { + let mut dir = tokio::fs::read_dir(dir).await?; + while let Some(entry) = dir.next_entry().await? { + if !entry.file_type().await?.is_file() { + continue; + } + if parse_transition_file_name(&entry.file_name().to_string_lossy()).is_none() { + tokio::fs::remove_file(entry.path()).await?; + } + } + Ok(()) +} + +fn parse_transition_file_name(name: &str) -> Option { + let (from, to) = name.split_once('_')?; + let mut transition = RealmTransition { from_root: [0; 32], to_root: [0; 32] }; + if hex::decode_to_slice(from, &mut transition.from_root).is_ok() + && hex::decode_to_slice(to, &mut transition.to_root).is_ok() + { + Some(transition) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use psy_data::p2p::{encode_proposal_body, proposal_from_parts, sha256, MAX_FINALIZER_OUTPUT_BYTES}; + + fn sample_object(salt: u8) -> (Proposal, Vec) { + let output = vec![salt; MAX_FINALIZER_OUTPUT_BYTES]; + let proof = vec![0xAB; 32]; + let mut updates = vec![0; STATE_UPDATES_ROOTS_OFFSET + 64 + 20]; + updates[40..72].fill(1); + updates[72..104].fill(2); + let body = encode_proposal_body(&output, &proof, &updates, &[0x11; 32]).unwrap(); + let proposal = proposal_from_parts(1, 0, 99, 1, [salt; 32], sha256(&output), sha256(&proof), sha256(&updates), sha256(&body)); + (proposal, body) + } + + #[tokio::test] + async fn transition_record_round_trip_and_read_bounds() { + let dir = tempfile::tempdir().unwrap(); + let (proposal, body) = sample_object(1); + let store = ProposalStore::open(dir.path()).await.unwrap(); + store.save_proposal(&proposal, &body).await.unwrap(); + drop(store); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let loaded = store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap(); + assert_eq!(loaded.0.proposal_id, proposal.proposal_id); + assert_eq!(loaded.1, body); + let mut request = BodyChunkRequest { proposal_id: proposal.proposal_id, offset: 0, max_bytes: 64 }; + assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); + let path = store.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); + let moved = dir.path().join("cached-object"); + tokio::fs::rename(&path, &moved).await.unwrap(); + request.offset = 64; + let second = store.read_body_chunk(&request).await.unwrap(); + assert_eq!(second.data, body[64..128]); + assert_eq!(second.body_hash, proposal.body_hash); + assert_eq!(second.body_len, body.len() as u64); + tokio::fs::rename(&moved, &path).await.unwrap(); + request.offset = body.len() as u64; + let end = store.read_body_chunk(&request).await.unwrap(); + assert!(end.eof); + assert!(end.data.is_empty()); + request.offset += 1; + assert!(store.read_body_chunk(&request).await.is_err()); + request.offset = 0; + request.max_bytes = 0; + assert!(store.read_body_chunk(&request).await.is_err()); + request.max_bytes = BODY_CHUNK_MAX_BYTES + 1; + assert!(store.read_body_chunk(&request).await.is_err()); + } + + #[tokio::test] + async fn staging_order_and_slot_replacement() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (first, first_body) = sample_object(1); + let (second, second_body) = sample_object(2); + let first_stage = store.create_staged(&first, &first_body).await.unwrap(); + let second_stage = store.create_staged(&second, &second_body).await.unwrap(); + assert!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().is_none()); + assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert_eq!(store.read_staged(&second_stage).await.unwrap().1, second_body); + store.install(second_stage).await.unwrap(); + assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![second.clone()]); + assert_eq!(store.read_staged(&first_stage).await.unwrap().1, first_body); + store.install(first_stage).await.unwrap(); + assert_eq!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().0.proposal_id, first.proposal_id); + assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![first.clone()]); + let replaced = BodyChunkRequest { proposal_id: second.proposal_id, offset: 0, max_bytes: 64 }; + assert!(store.read_body_chunk(&replaced).await.is_err()); + store.save_proposal(&second, &second_body).await.unwrap(); + assert_eq!(store.read_body_chunk(&replaced).await.unwrap().data, second_body[..64]); + store.save_proposal(&first, &first_body).await.unwrap(); + assert_eq!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().1, first_body); + } + + #[tokio::test] + async fn damaged_transition_is_dropped_and_reinstallable() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (proposal, body) = sample_object(1); + store.save_proposal(&proposal, &body).await.unwrap(); + let path = store.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); + let mut corrupted = encode_object(&proposal, &body); + *corrupted.last_mut().unwrap() ^= 1; + tokio::fs::write(&path, corrupted).await.unwrap(); + let request = BodyChunkRequest { proposal_id: proposal.proposal_id, offset: 0, max_bytes: 64 }; + // Cache hit: the installed header is served without touching disk. + assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![proposal.clone()]); + // The verified load is the corruption discovery point: the damaged body + // is dropped (record + file) so the pair returns to absent. + assert!(store.load_proposal(&[1; 32], &[2; 32]).await.is_err()); + assert!(!path_exists(&path).await); + assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert!(store.read_body_chunk(&request).await.is_err()); + let stage = store.create_staged(&proposal, &body).await.unwrap(); + assert_eq!(store.read_staged(&stage).await.unwrap().1, body); + store.install(stage).await.unwrap(); + assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); + tokio::fs::write(&path, &[]).await.unwrap(); + assert!(store.read_body_chunk(&request).await.is_err()); + assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert!(!path_exists(&path).await); + store.save_proposal(&proposal, &body).await.unwrap(); + assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); + } + + async fn path_exists(path: &str) -> bool { + tokio::fs::metadata(path).await.is_ok() + } +} diff --git a/psy_node_common/src/realm/processor/recovery.rs b/psy_node_common/src/realm/processor/recovery.rs new file mode 100644 index 00000000..19247d5f --- /dev/null +++ b/psy_node_common/src/realm/processor/recovery.rs @@ -0,0 +1,2384 @@ +//! Historical proposal verification and baseline FFS replay. + +use std::collections::{HashMap, HashSet}; + +use anyhow::Context; +use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; +use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; +use parth_core::{ + crypto::hash::{ + merkle_proof::MerkleProofCore, + traits::{FieldQHasher, HashTo4Felts, MerkleHasher, MerkleZeroHasher, QFieldHashable}, + }, + data::hash::{ + fast_node_serializer::{ + QMerkleStoreFastDoubleNodeSerializer, QMerkleStoreFastSingleNodeSerializer, + QMerkleStoreFastZeroNodeSerializer, QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, + }, + merkle_node_key::SimpleMerkleNodeKey, + }, + felt::{FromPrimitiveValuesFelt, QFelt64, ToU64Value}, + protocol::core_types::{Q256BitHash, QFHashBase, QNetworkTypesConfig}, +}; +use psy_config::CHECKPOINTS_PER_EPOCH; +use psy_core::job::job_id::{ProvingJobCircuitType, QProvingJobDataID}; +use psy_data::{ + p2p::{ + sha256, BodyChunkRequest, BodyChunkResponse, NodeId, Proposal, ProposalLookupEntry, + ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, + BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, + PROPOSAL_LOOKUP_CONCURRENCY, PROPOSAL_LOOKUP_ROUND_SECS, PROPOSAL_LOOKUP_TIMEOUT_SECS, + PROPOSAL_LOOKUP_WINDOW_PAIRS, + }, + prepared_block::realm::{PsyPreparedRealmBlockStateUpdates, PsyRealmCoordinatorUpdate}, + v1::qdata::{ + contract::{ + deserialize_imt_leaf_ffs_entry_v2, IMTContractStateLeaf, IMT_LEAF_FFS_ENTRY_SIZE_V2, + }, + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, + user::PQEDUserLeaf, + }, +}; +use psy_io::tokio::TokioLikeFileSystem; +use psy_node_core::{ + p2p::{traits::realm_coordinantor::RealmCoordinatorClient, validator_lookup::load_realm_validators_from_tree}, + psy_core_db::traits::full::{ + PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, + }, + psy_temp_db::StandardProcessorTempDBStoreBase, + queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, + store::traits::proof_store::QParthProofStore, +}; +use psy_serialize::PsyCanonicalDatabaseSerializeBaseSingle; + +use crate::{ + backup::realm::load_realm_memory_trees_from_db, + realm::{ + network::{NetworkError, RealmNetworkCommands}, + processor::{ + consensus::{ + decode_proposal_state_updates, validator_tree_root_matches_proof_base, + verify_proposal_submission, require_declared_roots_match_zk_output, + }, + db::PsyRealmDatabaseProcessor, + proposal_store::{ProposalStore, StagedProposal}, + }, + }, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CheckpointIdentity { + pub checkpoint_id: u64, + pub checkpoint_hash: [u8; 32], +} + +pub struct BaselineReplayRequest { + pub previous_checkpoint_id: u64, + pub updates: PsyPreparedRealmBlockStateUpdates, + pub reply: tokio::sync::oneshot::Sender>, +} + +pub struct VerifiedHistoryCandidate { + pub updates: PsyPreparedRealmBlockStateUpdates, + pub state_updates: Vec, + pub coordinator_update: PsyRealmCoordinatorUpdate, +} + +fn history_error(kind: &str, checkpoint_id: u64, detail: impl std::fmt::Display) -> anyhow::Error { + anyhow::anyhow!("{kind} at C={checkpoint_id}: {detail}") +} + +/// Why one history transition failed to verify: local material is absent or +/// stale (wait and retry), or the fetched candidate failed validation (prune +/// it and try the next candidate). +#[derive(Debug, thiserror::Error)] +pub(crate) enum RecoveryError { + #[error("{reason}")] + MissingLocalState { + #[source] + reason: anyhow::Error, + }, + #[error("{reason}")] + InvalidCandidate { + proposal_id: [u8; 32], + #[source] + reason: anyhow::Error, + }, +} + +pub(crate) fn invalid_candidate_id(error: &anyhow::Error) -> Option<[u8; 32]> { + match error.downcast_ref::()? { + RecoveryError::InvalidCandidate { proposal_id, .. } => Some(*proposal_id), + RecoveryError::MissingLocalState { .. } => None, + } +} + +fn missing_local_state(checkpoint_id: u64, detail: impl std::fmt::Display) -> RecoveryError { + RecoveryError::MissingLocalState { + reason: history_error("MissingHistoryProof", checkpoint_id, detail), + } +} + +fn gut_local_key( + key: SimpleMerkleNodeKey, + coordinator_height: u8, + realm_id: u64, +) -> anyhow::Result { + anyhow::ensure!( + key.level >= coordinator_height, + "InvalidStateUpdates: GUT node level {} is below coordinator height {}", + key.level, + coordinator_height + ); + let local_level = key.level - coordinator_height; + let expected_realm_id = if local_level >= 64 { + anyhow::ensure!(key.index == 0, "InvalidStateUpdates: GUT node index does not fit local level"); + 0 + } else { + key.index >> local_level + }; + anyhow::ensure!( + expected_realm_id == realm_id, + "InvalidStateUpdates: GUT node realm {expected_realm_id} does not match {realm_id}" + ); + let local_index = if local_level == 0 { + 0 + } else if local_level >= 64 { + key.index + } else { + key.index & ((1u64 << local_level) - 1) + }; + Ok(SimpleMerkleNodeKey { + level: local_level, + index: local_index, + }) +} + +fn require_width(bytes: &[u8], width: usize, what: &str) -> anyhow::Result<()> { + anyhow::ensure!( + bytes.is_empty() || bytes.len() % width == 0, + "InvalidStateUpdates: {what} length {} is not a multiple of {width}", + bytes.len() + ); + Ok(()) +} + +fn decode_double_id_node_ffs( + bytes: &[u8], + width: usize, + mut visit: impl FnMut(&[u8]), +) -> anyhow::Result<()> { + require_width(bytes, width, "tree node FFS")?; + for chunk in bytes.chunks_exact(width) { + visit(chunk); + } + Ok(()) +} + +fn seed_tree_from_merkle_proof( + tree: &mut SimpleMemoryMerkleRecorderStore, + proof: &MerkleProofCore, +) -> anyhow::Result<()> +where + H: MerkleZeroHasher + MerkleHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + anyhow::ensure!( + proof.verify::(), + "MissingAuthenticatedState: previous merkle proof does not verify" + ); + let mut key = SimpleMerkleNodeKey::new(tree.get_height(), proof.index); + for sibling in &proof.siblings { + tree.set_node_value(key.sibling(), *sibling); + key = key.parent(); + } + tree.set_leaf(proof.index, proof.value); + Ok(()) +} + +fn require_declared_double_id_nodes_match( + tree: &SimpleMemoryMerkleRecorderStore, + nodes: &[(u8, u64, Hash)], +) -> anyhow::Result<()> +where + H: MerkleZeroHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + for (level, index, value) in nodes { + let key = SimpleMerkleNodeKey { + level: *level, + index: *index, + }; + anyhow::ensure!( + tree.get_node_value(&key) == *value, + "InvalidStateUpdates: declared tree node {:?}={:?} does not match recomputed {:?}", + key, + value, + tree.get_node_value(&key) + ); + } + Ok(()) +} + +fn double_id_leaves_at_level( + nodes: &[(u8, u64, Hash)], + height: u8, +) -> HashMap { + let mut last_leaf = HashMap::new(); + for (level, index, value) in nodes { + if *level == height { + last_leaf.insert(*index, *value); + } + } + last_leaf +} + +pub fn replay_double_id_nodes_from_leaves( + tree: &mut SimpleMemoryMerkleRecorderStore, + nodes: &[(u8, u64, Hash)], +) -> anyhow::Result<()> +where + H: MerkleZeroHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + let height = tree.get_height(); + for (index, value) in double_id_leaves_at_level(nodes, height) { + tree.set_leaf(index, value); + } + require_declared_double_id_nodes_match(tree, nodes) +} + +fn require_imt_leaf_ffs_consistency(bytes: &[u8]) -> anyhow::Result<()> +where + F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, + Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, + H: MerkleZeroHasher + FieldQHasher, +{ + require_width(bytes, IMT_LEAF_FFS_ENTRY_SIZE_V2, "IMT leaf FFS")?; + let mut first: HashMap<(u64, u64, u64), ([u8; 32], bool)> = HashMap::new(); + let mut first_key_index: HashMap<(u64, u64, [u8; 32]), u64> = HashMap::new(); + let mut first_new_keys: HashSet<(u64, u64, [u8; 32])> = HashSet::new(); + for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { + let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, leaf_value, next_key, next_index, is_new_key) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + let leaf = IMTContractStateLeaf:: { + key: Hash::from_owned_32bytes(leaf_key), + value: Hash::from_owned_32bytes(leaf_value), + next_key: Hash::from_owned_32bytes(next_key), + next_index: F::from_u64_value(next_index), + }; + anyhow::ensure!( + leaf.qfhash::().into_owned_32bytes() == leaf_hash, + "InvalidStateUpdates: IMT leaf preimage does not bind leaf_hash" + ); + let id = (tree_id, tree_sub_id, leaf_index); + let key_id = (tree_id, tree_sub_id, leaf_key); + if let Some(first_index) = first_key_index.get(&key_id) { + if *first_index != leaf_index { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + } else { + first_key_index.insert(key_id, leaf_index); + } + if let Some((first_key, first_new)) = first.get(&id) { + if is_new_key && !first_new_keys.contains(&key_id) { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + if !*first_new && is_new_key { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + if *first_key != leaf_key { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + } else { + first.insert(id, (leaf_key, is_new_key)); + if is_new_key { + first_new_keys.insert(key_id); + } + } + } + let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], u64)> = HashMap::new(); + let mut seen_final = HashSet::new(); + for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { + let (tree_id, tree_sub_id, leaf_index, _, leaf_key, _, next_key, next_index, _) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + if seen_final.insert((tree_id, tree_sub_id, leaf_index)) { + finals.insert((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)); + } + } + for ((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)) in &finals { + if *next_index == 0 { + anyhow::ensure!( + *next_key == [0u8; 32], + "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + let Some((successor_key, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) else { + continue; + }; + anyhow::ensure!( + successor_key == next_key, + "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index} next_index={next_index}" + ); + anyhow::ensure!( + successor_key != leaf_key || *next_index == *leaf_index, + "InvalidStateUpdates: IMT successor key collides with source user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } + Ok(()) +} + +/// Changed contract-state leaves keyed by (user, contract, index). +pub fn contract_state_leaves_from_ffs( + updates: &PsyPreparedRealmBlockStateUpdates, +) -> anyhow::Result> +where + Hash: Copy + Q256BitHash, +{ + let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); + let mut contract_state_leaves: HashMap<(u64, u64, u64), Hash> = HashMap::new(); + if !updates.update_contract_state_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( + chunk, + ); + grouped + .entry((node.key.tree_id, node.key.tree_sub_id)) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + for ((user_id, contract_id), nodes) in grouped { + let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); + for (index, value) in double_id_leaves_at_level(&nodes, height) { + contract_state_leaves.insert((user_id, contract_id, index), value); + } + } + } + Ok(contract_state_leaves) +} + +pub fn require_state_update_record_coverage( + updates: &PsyPreparedRealmBlockStateUpdates, + checkpoint_id: u64, + imt_managed: &HashSet<(u64, u64, u64)>, +) -> anyhow::Result<()> +where + Hash: Copy + Q256BitHash, +{ + require_width( + &updates.update_global_user_tree_nodes_ffs, + QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, + "global user tree FFS", + )?; + require_width( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + "user contract tree FFS", + )?; + require_width( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + "contract state tree FFS", + )?; + require_width( + &updates.update_user_leaves_ffs, + PSY_OBJECT_FFS_SIZE_USER_LEAF, + "user leaf FFS", + )?; + require_width( + &updates.update_contract_state_imt_leaves_ffs, + IMT_LEAF_FFS_ENTRY_SIZE_V2, + "IMT leaf FFS", + )?; + if checkpoint_id == 0 { + return Ok(()); + } + let contract_state_leaves = contract_state_leaves_from_ffs(updates)?; + let mut imt_leaves: HashSet<(u64, u64, u64)> = HashSet::new(); + for chunk in updates + .update_contract_state_imt_leaves_ffs + .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) + { + let (tree_id, tree_sub_id, leaf_index, _, _, _, _, _, _) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + imt_leaves.insert((tree_id, tree_sub_id, leaf_index)); + } + for (user_id, contract_id, index) in imt_managed { + anyhow::ensure!( + imt_leaves.contains(&(*user_id, *contract_id, *index)), + "InvalidStateUpdates: contract-state leaf user={user_id} contract={contract_id} index={index} has no IMT record" + ); + } + let mut user_contract_leaves: HashSet<(u64, u64)> = HashSet::new(); + if !updates.update_user_contract_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + grouped + .entry(node.key.tree_id) + .or_default() + .push((node.key.level, node.key.index)); + }, + )?; + for (user_id, nodes) in grouped { + let height = nodes.iter().map(|(level, _)| *level).max().unwrap_or(0); + for (level, index) in nodes { + if level == height { + user_contract_leaves.insert((user_id, index)); + } + } + } + } + let contract_pairs: HashSet<(u64, u64)> = contract_state_leaves + .keys() + .map(|(user_id, contract_id, _)| (*user_id, *contract_id)) + .collect(); + for (user_id, contract_id) in &user_contract_leaves { + anyhow::ensure!( + contract_pairs.contains(&(*user_id, *contract_id)), + "InvalidStateUpdates: user-contract leaf user={user_id} contract={contract_id} has no contract-state FFS" + ); + } + Ok(()) +} + +pub fn replay_state_updates_into_tree( + tree: &mut SimpleMemoryMerkleRecorderStore, + updates: &PsyPreparedRealmBlockStateUpdates, + coordinator_height: u8, + realm_user_tree_height: u8, + realm_id: u64, + checkpoint_id: u64, + imt_managed: &HashSet<(u64, u64, u64)>, +) -> anyhow::Result<()> +where + F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, + Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, + H: MerkleZeroHasher + FieldQHasher, +{ + require_state_update_record_coverage(updates, checkpoint_id, imt_managed)?; + anyhow::ensure!( + tree.get_root() == updates.old_realm_root, + "MissingAuthenticatedState: tree root {:?} is not old_realm_root {:?}", + tree.get_root(), + updates.old_realm_root + ); + + let gut_nodes = if updates.update_global_user_tree_nodes_ffs.is_empty() { + Vec::new() + } else { + QMerkleStoreFastZeroNodeSerializer::deserialize_zero_id_nodes_from_slice::( + &updates.update_global_user_tree_nodes_ffs, + ) + }; + let min_user_id = realm_id << realm_user_tree_height; + let mut last_user: HashMap> = HashMap::new(); + for bytes in updates + .update_user_leaves_ffs + .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) + { + let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; + last_user.insert(leaf.user_id.to_u64_value(), leaf); + } + let mut last_leaf: HashMap = HashMap::new(); + for node in &gut_nodes { + let local = gut_local_key(node.key, coordinator_height, realm_id)?; + if local.level == realm_user_tree_height { + last_leaf.insert(local.index, node.value); + } + } + for (index, value) in &last_leaf { + let previous_leaf = tree.get_leaf_value(*index); + if previous_leaf == *value { + continue; + } + let user_id = min_user_id + *index; + let leaf = last_user.get(&user_id).ok_or_else(|| { + anyhow::anyhow!("InvalidStateUpdates: GUT leaf {index} changed without preimage") + })?; + anyhow::ensure!( + leaf.qfhash::() == *value, + "InvalidStateUpdates: user {user_id} preimage does not bind the GUT leaf" + ); + } + for (index, value) in last_leaf { + tree.set_leaf(index, value); + } + + for node in &gut_nodes { + let local = gut_local_key(node.key, coordinator_height, realm_id)?; + anyhow::ensure!( + tree.get_node_value(&local) == node.value, + "InvalidStateUpdates: declared GUT node {:?}={:?} does not match recomputed {:?}", + local, + node.value, + tree.get_node_value(&local) + ); + } + + for (user_id, leaf) in &last_user { + anyhow::ensure!( + *user_id >= min_user_id, + "InvalidStateUpdates: user_id {user_id} is outside realm {realm_id}" + ); + let local_index = user_id - min_user_id; + let expected = leaf.qfhash::(); + anyhow::ensure!( + tree.get_leaf_value(local_index) == expected, + "InvalidStateUpdates: user {user_id} preimage does not bind the recomputed GUT leaf" + ); + } + + require_imt_leaf_ffs_consistency::(&updates.update_contract_state_imt_leaves_ffs)?; + + anyhow::ensure!( + tree.get_root() == updates.new_realm_root, + "InvalidStateUpdates: recomputed root {:?} is not new_realm_root {:?}", + tree.get_root(), + updates.new_realm_root + ); + Ok(()) +} + +/// Non-empty changed leaves of every contract-state tree the previous +/// checkpoint's IMT index manages: those must ship IMT records — including +/// first-time inserts, whose `is_new_key` write is mandatory — or the IMT +/// index keeps proving a stale value or misses the key entirely. Trees the +/// IMT index has no entries for are positional and fully exempt; a leaf +/// cleared to zero keeps the no-IMT-required behavior. The next-append +/// pointer is read before this block's IMT FFS applies, so it still reflects +/// the state the update builds on. +pub async fn imt_managed_leaves_from_db( + db: &S, + updates: &PsyPreparedRealmBlockStateUpdates, +) -> anyhow::Result> +where + S: psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader + Sync, + F: parth_core::felt::QFelt64, + Hash: Q256BitHash + Copy + PartialEq, +{ + let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); + let mut changed_trees: HashMap<(u64, u64), Vec> = HashMap::new(); + for ((user_id, contract_id, index), new_value) in contract_state_leaves_from_ffs(updates)? { + if new_value == empty_leaf { + continue; + } + changed_trees + .entry((user_id, contract_id)) + .or_default() + .push(index); + } + let mut managed = HashSet::new(); + for ((user_id, contract_id), leaves) in changed_trees { + let next_append_index = db + .contract_state_imt_get_next_append_index(user_id, contract_id) + .await + .with_context(|| format!("previous-checkpoint IMT append index read failed user={user_id} contract={contract_id}"))?; + if next_append_index == 0 { + continue; + } + for index in leaves { + managed.insert((user_id, contract_id, index)); + } + } + Ok(managed) +} + +async fn load_previous_contract_heights( + db: &S, + previous_checkpoint_id: u64, + contract_ids: impl IntoIterator, +) -> anyhow::Result> +where + S: psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader + Sync, + F: Send + Sync, + Hash: Send + Sync, +{ + let mut unique = Vec::new(); + let mut seen = HashSet::new(); + for contract_id in contract_ids { + if seen.insert(contract_id) { + unique.push(contract_id); + } + } + if unique.is_empty() { + return Ok(HashMap::new()); + } + let fetched = db + .get_contract_tree_heights(previous_checkpoint_id, &unique) + .await + .map_err(|error| { + anyhow::anyhow!( + "MissingAuthenticatedState: contract heights unavailable at previous checkpoint {previous_checkpoint_id}: {error}" + ) + })?; + let mut heights = HashMap::with_capacity(unique.len()); + for (i, contract_id) in unique.into_iter().enumerate() { + heights.insert(contract_id, fetched.get(i).copied().unwrap_or(0)); + } + Ok(heights) +} + +fn require_previous_contract_height(heights: &HashMap, contract_id: u64) -> anyhow::Result { + let height = heights.get(&contract_id).copied().unwrap_or(0); + anyhow::ensure!( + height > 0, + "MissingAuthenticatedState: contract {contract_id} height is zero" + ); + Ok(height) +} + + + + + +impl< + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, + STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter + + PsyNodeCoreRewardsTagTreeStoreReader + + Send + + Sync, + GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync, + ProofWorkQueue: QStandardWorkerQueuePublisher + Send + Sync, + TempDatabase: StandardProcessorTempDBStoreBase, + ProofStore: QParthProofStore, + FileSystem: TokioLikeFileSystem + Send + Sync + 'static, + CoordinatorClient: RealmCoordinatorClient + Send + Sync, + > + PsyRealmDatabaseProcessor< + N, + S, + STagTreeRewards, + GUTAUpdateQueue, + ProofWorkQueue, + TempDatabase, + ProofStore, + FileSystem, + CoordinatorClient, + > +where + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, +{ + pub async fn verify_state_updates_from_baseline( + &self, + previous_checkpoint_id: u64, + updates: &PsyPreparedRealmBlockStateUpdates, + ) -> anyhow::Result<()> { + let imt_managed = crate::realm::processor::recovery::imt_managed_leaves_from_db::(self.db.as_ref(), updates) + .await?; + let mut trees = load_realm_memory_trees_from_db::( + &*self.db, + previous_checkpoint_id, + self.state.realm_id_u64, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState at previous checkpoint {previous_checkpoint_id}: {error}"))?; + let mut tree = trees.into_tuple().0; + let checkpoint_id = previous_checkpoint_id.saturating_add(1); + replay_state_updates_into_tree::( + &mut tree, + updates, + N::COORDINATOR_GLOBAL_USER_TREE_HEIGHT, + N::REALM_GLOBAL_USER_TREE_HEIGHT, + self.state.realm_id_u64, + checkpoint_id, + &imt_managed, + ) + .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}"))?; + self.verify_double_id_trees_from_previous(previous_checkpoint_id, updates) + .await + .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}")) + } + + async fn verify_double_id_trees_from_previous( + &self, + previous_checkpoint_id: u64, + updates: &PsyPreparedRealmBlockStateUpdates, + ) -> anyhow::Result<()> { + let mut last_user: HashMap> = HashMap::new(); + for bytes in updates + .update_user_leaves_ffs + .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) + { + let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; + last_user.insert(leaf.user_id.to_u64_value(), leaf); + } + let mut user_contract_leaves: HashMap<(u64, u64), N::QHash> = HashMap::new(); + if !updates.update_user_contract_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + grouped + .entry(node.key.tree_id) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + for (user_id, nodes) in grouped { + anyhow::ensure!( + nodes.iter().all(|(level, _, _)| *level <= N::GLOBAL_CONTRACT_TREE_HEIGHT), + "InvalidStateUpdates: user-contract node level exceeds tree height" + ); + let mut tree = SimpleMemoryMerkleRecorderStore::::new( + N::GLOBAL_CONTRACT_TREE_HEIGHT, + ); + let leaves = double_id_leaves_at_level(&nodes, N::GLOBAL_CONTRACT_TREE_HEIGHT); + if leaves.is_empty() && !nodes.is_empty() { + anyhow::bail!( + "MissingAuthenticatedState: user-contract internals for user {user_id} have no leaf preimages" + ); + } + for index in leaves.keys() { + let proof = self + .db + .user_contract_tree_get_merkle_proof(previous_checkpoint_id, user_id, *index) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract proof user={user_id} index={index}: {error}"))?; + seed_tree_from_merkle_proof(&mut tree, &proof)?; + } + replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; + for (index, value) in leaves { + user_contract_leaves.insert((user_id, index), value); + } + let new_root = tree.get_root(); + let bound = if let Some(leaf) = last_user.get(&user_id) { + leaf.user_state_tree_root + } else { + self.db + .get_user_leaf(previous_checkpoint_id, user_id) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user {user_id} leaf unavailable: {error}"))? + .user_state_tree_root + }; + anyhow::ensure!( + new_root == bound, + "InvalidStateUpdates: user {user_id} contract tree root does not bind the user leaf" + ); + } + } + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, N::QHash)>> = HashMap::new(); + if !updates.update_contract_state_tree_nodes_ffs.is_empty() { + decode_double_id_node_ffs( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( + chunk, + ); + grouped + .entry((node.key.tree_id, node.key.tree_sub_id)) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + } + let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)> = HashMap::new(); + if !updates.update_contract_state_imt_leaves_ffs.is_empty() { + for chunk in updates + .update_contract_state_imt_leaves_ffs + .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) + { + let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, _, next_key, next_index, is_new_key) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + finals.entry((tree_id, tree_sub_id, leaf_index)).or_insert(( + leaf_hash, + leaf_key, + next_key, + next_index, + is_new_key, + )); + } + } + let heights = load_previous_contract_heights( + self.db.as_ref(), + previous_checkpoint_id, + grouped + .keys() + .map(|(_, contract_id)| *contract_id) + .chain(finals.keys().map(|(_, contract_id, _)| *contract_id)), + ) + .await?; + let mut contract_state_leaves: HashMap<(u64, u64, u64), N::QHash> = HashMap::new(); + for ((user_id, contract_id), nodes) in grouped { + let height = require_previous_contract_height(&heights, contract_id)?; + anyhow::ensure!( + nodes.iter().all(|(level, _, _)| *level <= height), + "InvalidStateUpdates: contract-state node level exceeds authenticated height {height}" + ); + let mut tree = SimpleMemoryMerkleRecorderStore::::new(height); + let leaves = double_id_leaves_at_level(&nodes, height); + if leaves.is_empty() && !nodes.is_empty() { + anyhow::bail!( + "MissingAuthenticatedState: contract-state internals for user={user_id} contract={contract_id} have no leaf preimages" + ); + } + for index in leaves.keys() { + let proof = self + .db + .contract_state_tree_get_merkle_proof( + previous_checkpoint_id, + user_id, + contract_id, + height, + *index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: contract-state proof user={user_id} contract={contract_id} index={index}: {error}"))?; + seed_tree_from_merkle_proof(&mut tree, &proof)?; + } + replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; + for (index, value) in leaves { + contract_state_leaves.insert((user_id, contract_id, index), value); + } + let new_root = tree.get_root(); + let bound = if let Some(leaf) = user_contract_leaves.get(&(user_id, contract_id)) { + *leaf + } else { + self.db + .user_contract_tree_get_leaf_hash(previous_checkpoint_id, user_id, contract_id) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract leaf user={user_id} contract={contract_id}: {error}"))? + }; + anyhow::ensure!( + new_root == bound, + "InvalidStateUpdates: contract-state root does not bind user {user_id} contract {contract_id}" + ); + } + self.verify_imt_from_previous(previous_checkpoint_id, finals, &contract_state_leaves, &heights) + .await + } + + async fn verify_imt_from_previous( + &self, + previous_checkpoint_id: u64, + finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)>, + contract_state_leaves: &HashMap<(u64, u64, u64), N::QHash>, + heights: &HashMap, + ) -> anyhow::Result<()> { + for ((tree_id, tree_sub_id, leaf_index), (leaf_hash, leaf_key, next_key, next_index, is_new_key)) in &finals { + let height = require_previous_contract_height(heights, *tree_sub_id)?; + let expected = if let Some(value) = contract_state_leaves.get(&(*tree_id, *tree_sub_id, *leaf_index)) { + *value + } else { + self.db + .contract_state_tree_get_leaf_hash( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + height, + *leaf_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))? + }; + anyhow::ensure!( + expected.into_owned_32bytes() == *leaf_hash, + "InvalidStateUpdates: IMT leaf_hash does not bind contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + let key = N::QHash::from_owned_32bytes(*leaf_key); + let old_index = self + .db + .contract_state_imt_get_leaf_index_for_key(previous_checkpoint_id, *tree_id, *tree_sub_id, &key) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT key index user={tree_id} contract={tree_sub_id}: {error}"))?; + if *is_new_key { + anyhow::ensure!( + old_index.is_none(), + "InvalidStateUpdates: IMT is_new_key already indexed user={tree_id} contract={tree_sub_id}" + ); + let previous_at_index = self + .db + .contract_state_imt_get_leaf_preimage( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + *leaf_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT previous leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))?; + anyhow::ensure!( + previous_at_index.is_none(), + "InvalidStateUpdates: IMT new key overwrites an authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } else { + let old_index = old_index.ok_or_else(|| { + anyhow::anyhow!( + "InvalidStateUpdates: IMT key is not new but has no authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" + ) + })?; + anyhow::ensure!( + old_index == *leaf_index, + "InvalidStateUpdates: IMT key index moved user={tree_id} contract={tree_sub_id}" + ); + } + if *next_index == 0 { + anyhow::ensure!( + *next_key == [0u8; 32], + "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + if let Some((_, successor_key, _, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) { + anyhow::ensure!( + successor_key == next_key, + "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + let successor = self + .db + .contract_state_imt_get_leaf_preimage( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + *next_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT successor user={tree_id} contract={tree_sub_id} next_index={next_index}: {error}"))?; + let successor = successor.ok_or_else(|| { + anyhow::anyhow!("MissingAuthenticatedState: IMT successor missing user={tree_id} contract={tree_sub_id} next_index={next_index}") + })?; + anyhow::ensure!( + successor.key.into_owned_32bytes() == *next_key, + "InvalidStateUpdates: IMT next_key does not match authenticated successor user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } + Ok(()) + } + + pub async fn verify_history_proposal( + &self, + included: &CheckpointIdentity, + proposal: &Proposal, + body: &[u8], + ) -> anyhow::Result<( + PsyPreparedRealmBlockStateUpdates, + Vec, + PsyRealmCoordinatorUpdate, + )> { + anyhow::ensure!( + proposal.realm_id == self.state.realm_id_u64 as u32, + "InvalidStateUpdates at C={}: proposal realm mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + proposal.chain_id == self.state.chain_id, + "InvalidStateUpdates at C={}: proposal chain mismatch", + included.checkpoint_id + ); + let coordinator_update = self + .coordinator_client + .rc_get_realm_sync_info(included.checkpoint_id, self.state.realm_id_u64) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("Coordinator C materials unavailable: {error:#}"), + ))?; + anyhow::ensure!( + coordinator_update.checkpoint_sync_info.checkpoint_id == included.checkpoint_id, + "MissingHistoryProof at C={}: coordinator checkpoint id mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + coordinator_update + .checkpoint_sync_info + .checkpoint_leaf_hash + .into_owned_32bytes() + == included.checkpoint_hash, + "MissingHistoryProof at C={}: coordinator leaf hash does not match included.checkpoint_hash", + included.checkpoint_id + ); + let authenticated_leaf = self + .checkpoint_tree_backup_manager + .checkpoint_tree + .get_leaf(included.checkpoint_id); + if authenticated_leaf.value.into_owned_32bytes() != included.checkpoint_hash { + return Err(missing_local_state( + included.checkpoint_id, + "included.checkpoint_hash does not match the authenticated checkpoint tree leaf", + ) + .into()); + } + let roots = self + .db + .get_checkpoint_global_state_roots(proposal.base_checkpoint_id) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("proof-base P={} roots unavailable: {error:#}", proposal.base_checkpoint_id), + ))?; + anyhow::ensure!( + validator_tree_root_matches_proof_base( + &proposal.validator_tree_root, + &roots.validator_tree_root.into_owned_32bytes(), + ), + "MissingHistoryProof at C={}: proposal.validator_tree_root does not match P={}", + included.checkpoint_id, + proposal.base_checkpoint_id + ); + let (_, _, user_ids, _) = + load_realm_validators_from_tree::( + &*self.db, + self.state.chain_id, + proposal.base_checkpoint_id, + proposal.realm_id, + &roots.validator_tree_root, + ) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("proof-base P={} validator tree unavailable: {error:#}", proposal.base_checkpoint_id), + ))?; + let proposer_user_id = user_ids + .iter() + .find(|(sub_id, _)| *sub_id == proposal.proposer_sub_id) + .map(|(_, user_id)| *user_id) + .ok_or_else(|| { + history_error( + "MissingHistoryProof", + included.checkpoint_id, + format!("proposer sub_id {} is not a validator", proposal.proposer_sub_id), + ) + })?; + let decoded = + verify_proposal_submission::(proposal, body, proposer_user_id, self.proof_verifier.as_ref())?; + let updates = decode_proposal_state_updates::(&decoded.state_updates)?; + anyhow::ensure!( + updates.realm_id == proposal.realm_id as u64, + "InvalidStateUpdates at C={}: FFS realm_id {} does not match proposal {}", + included.checkpoint_id, + updates.realm_id, + proposal.realm_id + ); + anyhow::ensure!( + updates.realm_sub_id == proposal.proposer_sub_id as u64, + "InvalidStateUpdates at C={}: FFS realm_sub_id {} does not match proposer {}", + included.checkpoint_id, + updates.realm_sub_id, + proposal.proposer_sub_id + ); + let output = psy_data::guta::realm_finalize::protocol_decode_finalize_output::( + &decoded.output, + )?; + require_declared_roots_match_zk_output(&updates, &output)?; + anyhow::ensure!( + updates.new_realm_root.into_owned_32bytes() != [0u8; 32] + || updates.old_realm_root == updates.new_realm_root, + "InvalidStateUpdates at C={}: empty new root", + included.checkpoint_id + ); + let realm_proof = &coordinator_update.merkle_proof_to_realm_root; + anyhow::ensure!( + realm_proof.verify::(), + "MissingHistoryProof at C={}: realm-root path does not verify", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.index == self.state.realm_id_u64, + "MissingHistoryProof at C={}: realm-root path index mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.value == updates.new_realm_root, + "MissingHistoryProof at C={}: authenticated realm root does not match proposal new_realm_root", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.root == coordinator_update.checkpoint_sync_info.state_roots.user_tree_root, + "MissingHistoryProof at C={}: realm-root path is not bound to C user_tree_root", + included.checkpoint_id + ); + let previous = included + .checkpoint_id + .checked_sub(1) + .ok_or_else(|| history_error("MissingHistoryProof", included.checkpoint_id, "C=0 has no predecessor"))?; + self.verify_state_updates_from_baseline(previous, &updates) + .await + .with_context(|| { + format!( + "InvalidStateUpdates at C={} proposal_id={}", + included.checkpoint_id, + hex::encode(proposal.proposal_id) + ) + })?; + Ok((updates, decoded.state_updates, coordinator_update)) + } + + pub async fn ensure_uncommitted_processing_ids(&mut self, checkpoint_id: u64) -> anyhow::Result<()> { + let pending_id = self.state.processing_unique_pending_id; + let mapped_checkpoint = self.db.get_checkpoint_id_for_unique_pending_id(pending_id).await?; + if pending_id != 0 && mapped_checkpoint == Some(checkpoint_id) { + return Ok(()); + } + let (pending_id, proc_checkpoint_unique_id) = + if let Some(ids) = self.db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await? { + ids + } else if pending_id != 0 && mapped_checkpoint.is_none() { + return Ok(()); + } else { + self.db.inc_unique_pending_id(1).await? + }; + self.state.processing_unique_pending_id = pending_id; + self.state.processing_proc_checkpoint_unique_id = proc_checkpoint_unique_id; + self.temp_db + .set_unique_pending_ids(&self.state.realm_identifier, pending_id, proc_checkpoint_unique_id) + .await?; + Ok(()) + } + + pub async fn apply_history_proposal( + &mut self, + included: &CheckpointIdentity, + verified: VerifiedHistoryCandidate, + ) -> anyhow::Result<(PsyPreparedRealmBlockStateUpdates, Vec)> { + let VerifiedHistoryCandidate { + updates, + state_updates, + coordinator_update, + } = verified; + self.ensure_uncommitted_processing_ids(included.checkpoint_id).await?; + self.state.processing_checkpoint_id = included.checkpoint_id; + self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + self.state.processing_realm_start_root = updates.old_realm_root; + self.state.processing_realm_end_root = updates.new_realm_root; + if self.state.last_committed_checkpoint_id >= included.checkpoint_id { + anyhow::ensure!( + self.state.last_committed_realm_end_root == updates.new_realm_root, + "InvalidStateUpdates at C={}: committed realm root does not match candidate; refusing second FFS", + included.checkpoint_id + ); + } else { + self.commit_state( + &coordinator_update, + &updates, + ProvingJobCircuitType::GUTANoChange, + vec![], + ) + .await?; + } + Ok((updates, state_updates)) + } + + /// Verify the candidate for one root pair, reading it from the staged bytes + /// when the fetch stage supplied them and from the pair slot otherwise. + pub async fn verify_history_transition( + &self, + included: &CheckpointIdentity, + pair: RealmTransition, + staged: Option<&StagedProposal>, + store: &ProposalStore, + ) -> anyhow::Result>> { + let loaded = match staged { + Some(staged) => Some(store.read_staged(staged).await?), + None => store + .load_proposal(&pair.old_root, &pair.new_root) + .await?, + }; + let Some((proposal, body)) = loaded else { + return Ok(None); + }; + match self + .verify_history_proposal(included, &proposal, &body) + .await + { + Ok((updates, state_updates, coordinator_update)) => { + Ok(Some(VerifiedHistoryCandidate { + updates, + state_updates, + coordinator_update, + })) + } + Err(error) => { + if error + .downcast_ref::() + .is_some_and(|classified| matches!(classified, RecoveryError::MissingLocalState { .. })) + { + return Err(error); + } + tracing::warn!( + "history verify rejected C={} pair=({},{}) proposal={} error={error}", + included.checkpoint_id, + hex::encode(pair.old_root), + hex::encode(pair.new_root), + hex::encode(proposal.proposal_id) + ); + Err(anyhow::Error::new(RecoveryError::InvalidCandidate { + proposal_id: proposal.proposal_id, + reason: anyhow::anyhow!("history verify rejected C={}: {error:#}", included.checkpoint_id), + })) + } + } + } +} + +/// Attempts per checkpoint: the staged window candidate plus re-fetches that +/// exclude the candidates already rejected for that pair. +#[cfg(test)] +mod tests { + use super::*; + use crate::realm::processor::catchup::*; + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; + use parth_core::{ + crypto::hash::traits::{MerkleZeroHasher, QFieldHashable}, + data::hash::merkle_node_key::SimpleMerkleNode, + pgoldilocks::{PGoldilocksFelt, PGoldilocksHash, PoseidonHasher}, + protocol::core_types::Q256BitHash, + }; + use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdates; + + fn verify_double_id_node_ffs_bytes( + bytes: &[u8], + width: usize, + parse: impl Fn(&[u8]) -> (u64, u64, u8, u64, Hash), + ) -> anyhow::Result<()> + where + H: MerkleZeroHasher, + Hash: Q256BitHash + Copy + PartialEq + Default + std::fmt::Debug, + { + require_width(bytes, width, "tree node FFS")?; + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); + for chunk in bytes.chunks_exact(width) { + let (tree_id, tree_sub_id, level, index, value) = parse(chunk); + grouped.entry((tree_id, tree_sub_id)).or_default().push((level, index, value)); + } + for nodes in grouped.values() { + let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); + let mut tree = SimpleMemoryMerkleRecorderStore::::new(height.max(1)); + replay_double_id_nodes_from_leaves(&mut tree, nodes)?; + } + Ok(()) + } + + fn empty_updates(old: PGoldilocksHash, new: PGoldilocksHash) -> PsyPreparedRealmBlockStateUpdates { + PsyPreparedRealmBlockStateUpdates { + realm_id: 0, + realm_sub_id: 0, + unique_pending_id: 0, + proc_checkpoint_unique_id: Default::default(), + old_realm_root: old, + new_realm_root: new, + update_global_user_tree_nodes_ffs: vec![], + update_user_contract_tree_nodes_ffs: vec![], + update_contract_state_tree_nodes_ffs: vec![], + update_user_leaves_ffs: vec![], + update_contract_state_imt_leaves_ffs: vec![], + } + } + + #[test] + fn history_bad_ffs() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + let poison = SimpleMerkleNode { + key: SimpleMerkleNodeKey { level: 8, index: 0 }, + value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), + }; + updates.update_global_user_tree_nodes_ffs = + QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&poison); + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("poisoned GUT node must fail baseline replay"); + assert!( + error.to_string().contains("InvalidStateUpdates"), + "{error}" + ); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + let poison_contract = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x44; 32]), + }; + let mut contract_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&contract_leaf); + contract_ffs.extend_from_slice( + &QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&poison_contract), + ); + let leaf_poison = SimpleMerkleNode { + key: SimpleMerkleNodeKey { level: 16, index: 0 }, + value: PGoldilocksHash::from_owned_32bytes([0x22; 32]), + }; + let mut leaf_updates = empty_updates(old, old); + leaf_updates.update_global_user_tree_nodes_ffs = + QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&leaf_poison); + let preimage_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &leaf_updates, + 8, + 8, + 0, + 1, + &HashSet::new(), + ) + .expect_err("changed GUT leaf without preimage must fail"); + assert!( + preimage_error.to_string().contains("InvalidStateUpdates"), + "{preimage_error}" + ); + let contract_error = verify_double_id_node_ffs_bytes::( + &contract_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + (node.key.tree_id, 0, node.key.level, node.key.index, node.value) + }, + ) + .expect_err("poisoned contract node must fail baseline replay"); + assert!( + contract_error.to_string().contains("InvalidStateUpdates"), + "{contract_error}" + ); + } + + #[test] + fn history_duplicate_imt() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + let first_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ); + let second_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 7, true, + ); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&first_entry); + bytes.extend_from_slice(&second_entry); + updates.update_contract_state_imt_leaves_ffs = bytes; + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("conflicting IMT history must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + let moved = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 9, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ); + let mut moved_bytes = Vec::new(); + moved_bytes.extend_from_slice(&first_entry); + moved_bytes.extend_from_slice(&moved); + let mut moved_updates = empty_updates(old, old); + moved_updates.update_contract_state_imt_leaves_ffs = moved_bytes; + let moved_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &moved_updates, + 8, + 8, + 0, + 1, + &HashSet::new(), + ) + .expect_err("same IMT key at a second leaf index must fail"); + assert!(moved_error.to_string().contains("InvalidStateUpdates"), "{moved_error}"); + let successor_key = PGoldilocksHash::from_owned_32bytes([0x44u8; 32]); + let successor_leaf = IMTContractStateLeaf:: { + key: successor_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let successor_hash = successor_leaf.qfhash::(); + let successor = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 4, &successor_hash, &successor_key, &first_value, &first_next_key, 0, false, + ); + let mismatched_next = PGoldilocksHash::from_owned_32bytes([0x99u8; 32]); + let mismatched_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: mismatched_next, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(4), + }; + let mismatched_hash = mismatched_leaf.qfhash::(); + let mismatched = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &mismatched_hash, &first_key, &first_value, &mismatched_next, 4, false, + ); + let mut successor_bytes = Vec::new(); + successor_bytes.extend_from_slice(&mismatched); + successor_bytes.extend_from_slice(&successor); + let successor_error = require_imt_leaf_ffs_consistency::(&successor_bytes) + .expect_err("IMT next_key must match the successor leaf in the same FFS"); + assert!(successor_error.to_string().contains("InvalidStateUpdates"), "{successor_error}"); + } + + #[test] + fn history_ffs_coverage_requires_imt_for_changed_contract_leaf() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut updates = empty_updates(old, old); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + updates.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); + let error = require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect_err("changed contract-state leaf without IMT must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("no IMT record"), "{error}"); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + updates.update_contract_state_imt_leaves_ffs = + psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ) + .to_vec(); + require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect("matching IMT record must close the coverage set"); + require_state_update_record_coverage(&updates, 0, &HashSet::new()) + .expect("genesis may carry contract-state FFS with empty IMT"); + let mut genesis = empty_updates(old, old); + genesis.update_contract_state_tree_nodes_ffs = updates.update_contract_state_tree_nodes_ffs.clone(); + require_state_update_record_coverage(&genesis, 0, &HashSet::new()) + .expect("genesis contract-state leaves with empty IMT are a legal empty IMT"); + let genesis_error = require_state_update_record_coverage(&genesis, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect_err("the same missing IMT must fail after genesis"); + assert!(genesis_error.to_string().contains("no IMT record"), "{genesis_error}"); + require_state_update_record_coverage(&empty_updates(old, old), 1, &HashSet::new()) + .expect("empty contract-state FFS and empty IMT is a legal no-op"); + } + + #[test] + fn history_ffs_user_contract_requires_contract_state() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut updates = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 2, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + updates.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + let error = require_state_update_record_coverage(&updates, 1, &HashSet::new()) + .expect_err("user-contract leaf without contract-state FFS must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("no contract-state FFS"), "{error}"); + require_state_update_record_coverage(&updates, 0, &HashSet::new()) + .expect("genesis may register a user-contract leaf with empty contract-state and IMT"); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + updates.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + updates.update_contract_state_imt_leaves_ffs = + psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ) + .to_vec(); + require_state_update_record_coverage(&updates, 1, &HashSet::new()) + .expect("user-contract plus matching contract-state and IMT must close the coverage set"); + } + + #[test] + fn history_ffs_genesis_skips_imt_coverage() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut registered = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 262144, + level: 8, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + registered.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + require_state_update_record_coverage(®istered, 0, &HashSet::new()) + .expect("genesis contract registration has empty contract-state and IMT"); + let registered_error = require_state_update_record_coverage(®istered, 1, &HashSet::new()) + .expect_err("non-genesis registration without contract-state FFS must fail"); + assert!( + registered_error.to_string().contains("no contract-state FFS"), + "{registered_error}" + ); + let mut empty_value = empty_updates(old, old); + let empty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 262144, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), + }; + empty_value.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&empty_contract_leaf); + require_state_update_record_coverage(&empty_value, 0, &HashSet::new()) + .expect("genesis empty contract-state leaf with empty IMT is trusted setup"); + require_state_update_record_coverage(&empty_value, 1, &HashSet::new()) + .expect("new empty contract-state leaf does not require IMT"); + let mut nonempty = empty_updates(old, old); + let nonempty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 262144, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), + }; + nonempty.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&nonempty_contract_leaf); + require_state_update_record_coverage(&nonempty, 0, &HashSet::new()) + .expect("genesis non-empty contract-state leaf with empty IMT is trusted setup"); + let nonempty_error = require_state_update_record_coverage(&nonempty, 1, &HashSet::from_iter([(262144u64, 0u64, 0u64)])) + .expect_err("updated non-empty contract-state leaf without IMT must fail"); + assert!(nonempty_error.to_string().contains("no IMT record"), "{nonempty_error}"); + replay_state_updates_into_tree::( + &mut tree, &nonempty, 8, 8, 0, 0, &HashSet::new(), + ) + .expect("verify path must honor checkpoint_id=0 and skip IMT pairing"); + let verify_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &nonempty, + 8, + 8, + 0, + 1, + &HashSet::from_iter([(262144u64, 0u64, 0u64)]), + ) + .expect_err("verify path at C=1 must still require IMT for a non-empty leaf"); + assert!(verify_error.to_string().contains("no IMT record"), "{verify_error}"); + let mut poisoned = empty_updates(old, old); + poisoned.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; + let width_error = require_state_update_record_coverage(&poisoned, 0, &HashSet::new()) + .expect_err("genesis still rejects poisoned FFS widths"); + assert!(width_error.to_string().contains("user leaf FFS"), "{width_error}"); + } + + #[test] + fn history_ffs_positional_contract_leaf_needs_no_imt() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut positional = empty_updates(old, old); + let positional_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1310720, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + positional.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&positional_leaf); + require_state_update_record_coverage(&positional, 1, &HashSet::new()) + .expect("positional non-empty contract-state leaf with no previous IMT entry is exempt"); + let mut cleared = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 2, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + cleared.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + let zero_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), + }; + cleared.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&zero_leaf); + require_state_update_record_coverage(&cleared, 1, &HashSet::new()) + .expect("user-contract pairing survives a contract-state leaf cleared to zero"); + require_state_update_record_coverage(&cleared, 0, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect("genesis skips IMT coverage even with a declared managed set"); + } + + struct IMTPreimageFixture { + next_append: HashMap<(u64, u64), u64>, + } + + #[async_trait::async_trait] + impl psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader for IMTPreimageFixture { + async fn contract_state_imt_get_leaf_preimage( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _leaf_index: u64, + ) -> anyhow::Result>> { + Ok(None) + } + + async fn contract_state_imt_get_leaf_index_for_key( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _key: &PGoldilocksHash, + ) -> anyhow::Result> { + Ok(None) + } + + async fn contract_state_imt_find_predecessor( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _key: &PGoldilocksHash, + ) -> anyhow::Result<(u64, IMTContractStateLeaf)> { + Ok((0, IMTContractStateLeaf::default())) + } + + async fn contract_state_imt_get_next_append_index(&self, user_id: u64, contract_id: u64) -> anyhow::Result { + Ok(self.next_append.get(&(user_id, contract_id)).copied().unwrap_or(0)) + } + } + + fn contract_state_leaf_ffs(user_id: u64, contract_id: u64, index: u64, value_byte: u8) -> Vec { + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec( + &parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: user_id, + tree_sub_id: contract_id, + level: 4, + index, + }, + value: PGoldilocksHash::from_owned_32bytes([value_byte; 32]), + }, + ) + } + + struct CountingHeightStore { + heights: HashMap<(u64, u64), u8>, + calls: std::sync::atomic::AtomicUsize, + batches: std::sync::Mutex)>>, + fail: bool, + } + + impl CountingHeightStore { + fn new(heights: HashMap<(u64, u64), u8>) -> Self { + Self { + heights, + calls: std::sync::atomic::AtomicUsize::new(0), + batches: std::sync::Mutex::new(Vec::new()), + fail: false, + } + } + } + + #[async_trait::async_trait] + impl psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader< + PGoldilocksFelt, + PGoldilocksHash, + > for CountingHeightStore + { + async fn get_contract_tree_heights( + &self, + checkpoint_id: u64, + contract_ids: &[u64], + ) -> anyhow::Result> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.batches + .lock() + .expect("height batch log") + .push((checkpoint_id, contract_ids.to_vec())); + if self.fail { + anyhow::bail!("injected height store failure"); + } + Ok(contract_ids + .iter() + .map(|contract_id| self.heights.get(&(checkpoint_id, *contract_id)).copied().unwrap_or(0)) + .collect()) + } + } + + // Same C+I union the verify path feeds the loader: CST groups (1,7),(2,7),(1,8) + // then IMT finals (1,7,0),(1,9,0) with a duplicate (1,7,0) that or_insert keeps first. + const C_AND_I_IDS: [u64; 5] = [7, 7, 8, 7, 9]; + + #[tokio::test] + async fn previous_heights_batch_unique_c_and_i_at_historical_checkpoint() { + let previous = 10u64; + let store = CountingHeightStore::new(HashMap::from([ + ((previous, 7), 8), + ((previous, 8), 16), + ((previous, 9), 24), + ((previous + 1, 7), 32), + ])); + let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>( + &store, + previous, + C_AND_I_IDS, + ) + .await + .expect("batch heights"); + assert_eq!(store.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + let batches = store.batches.lock().expect("height batch log"); + assert_eq!(batches.as_slice(), &[(previous, vec![7, 8, 9])]); + drop(batches); + assert_eq!(require_previous_contract_height(&heights, 7).unwrap(), 8); + assert_eq!(require_previous_contract_height(&heights, 8).unwrap(), 16); + assert_eq!(require_previous_contract_height(&heights, 9).unwrap(), 24); + let later = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous + 1, [7]) + .await + .expect("later checkpoint is a different key"); + assert_eq!(require_previous_contract_height(&later, 7).unwrap(), 32); + } + + #[tokio::test] + async fn previous_heights_reject_zero_missing_and_injected_db_error() { + let previous = 10u64; + let store = CountingHeightStore::new(HashMap::from([((previous, 7), 0)])); + let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous, [7, 8]) + .await + .expect("missing maps to zero without a store error"); + let zero = require_previous_contract_height(&heights, 7).expect_err("zero height must reject"); + assert!(zero.to_string().contains("height is zero"), "{zero}"); + let missing = require_previous_contract_height(&heights, 8).expect_err("absent height must reject"); + assert!(missing.to_string().contains("height is zero"), "{missing}"); + let failing = CountingHeightStore { + fail: true, + ..CountingHeightStore::new(HashMap::new()) + }; + let error = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&failing, previous, [7]) + .await + .expect_err("store failure must reject"); + assert!(error.to_string().contains("MissingAuthenticatedState"), "{error}"); + assert!(error.to_string().contains("injected height store failure"), "{error}"); + assert_eq!(failing.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + + #[tokio::test] + async fn history_imt_managed_set_follows_tree_append_index() { + let mut managed_tree = IMTPreimageFixture { next_append: HashMap::new() }; + managed_tree.next_append.insert((1, 2), 2); + + // A first-time insert on an IMT-managed tree is NOT exempt: the is_new_key + // index write is mandatory, so the new slot must ship its IMT record too. + let mut new_key = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + new_key.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 5, 0x07); + let managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &new_key) + .await + .unwrap(); + assert_eq!(managed, HashSet::from_iter([(1u64, 2u64, 5u64)])); + let error = require_state_update_record_coverage(&new_key, 1, &managed) + .expect_err("new key on a managed tree without IMT record must fail"); + assert!(error.to_string().contains("no IMT record"), "{error}"); + + let mut tracked_changed = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + tracked_changed.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x05); + let tracked_managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &tracked_changed) + .await + .unwrap(); + assert_eq!(tracked_managed, HashSet::from_iter([(1u64, 2u64, 0u64)])); + + let positional_tree = IMTPreimageFixture { next_append: HashMap::new() }; + let untracked = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&positional_tree, &tracked_changed) + .await + .unwrap(); + assert!(untracked.is_empty(), "trees with no IMT entries are positional"); + require_state_update_record_coverage(&tracked_changed, 1, &untracked) + .expect("collector output for a positional leaf must pass coverage"); + + let mut cleared = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + cleared.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x00); + let cleared_managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &cleared) + .await + .unwrap(); + assert!(cleared_managed.is_empty(), "leaf cleared to zero keeps the no-IMT behavior"); + } + + + #[test] + fn history_imt_terminal_next_key_must_be_zero() { + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let nonzero_next = PGoldilocksHash::from_owned_32bytes([0x99; 32]); + let leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: nonzero_next, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let leaf_hash = leaf.qfhash::(); + let entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &leaf_hash, &first_key, &first_value, &nonzero_next, 0, false, + ); + let error = require_imt_leaf_ffs_consistency::(&entry) + .expect_err("terminal IMT next_index=0 with nonzero next_key must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("terminal next_key"), "{error}"); + } + + #[test] + fn history_poison_included_transition() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + updates.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("poisoned user-leaf width must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(!error.to_string().contains("auto")); + } + + fn sample_store_object(old_root: [u8; 32], new_root: [u8; 32], salt: u8) -> (psy_data::p2p::Proposal, Vec) { + sample_store_object_at(old_root, new_root, salt, 99, 1) + } + + fn sample_store_object_at( + old_root: [u8; 32], + new_root: [u8; 32], + salt: u8, + base_checkpoint_id: u64, + proposer_sub_id: u16, + ) -> (psy_data::p2p::Proposal, Vec) { + let output = vec![salt; psy_data::p2p::MAX_FINALIZER_OUTPUT_BYTES]; + let proof = vec![0xABu8; 32]; + let mut state_updates = vec![0u8; 40 + 64 + 20]; + state_updates[40..72].copy_from_slice(&old_root); + state_updates[72..104].copy_from_slice(&new_root); + let worker_tag = [0x11u8; 32]; + let body = psy_data::p2p::encode_proposal_body(&output, &proof, &state_updates, &worker_tag).unwrap(); + let proposal = psy_data::p2p::proposal_from_parts( + 1, + 0, + base_checkpoint_id, + proposer_sub_id, + [salt; 32], + psy_data::p2p::sha256(&output), + psy_data::p2p::sha256(&proof), + psy_data::p2p::sha256(&state_updates), + psy_data::p2p::sha256(&body), + ); + (proposal, body) + } + + #[tokio::test] + async fn history_ready_store_selection() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (proposal, body) = sample_store_object([1u8; 32], [2u8; 32], 1); + store.save_proposal(&proposal, &body).await.unwrap(); + let found = store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, proposal.proposal_id); + let loaded = store + .load_proposal(&[1u8; 32], &[2u8; 32]) + .await + .unwrap() + .expect("slot holds the stored pair"); + assert_eq!(loaded.0.proposal_id, proposal.proposal_id); + assert_eq!(loaded.1, body); + } + + #[tokio::test] + async fn history_ready_single_body_per_pair() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (first, first_body) = sample_store_object([9u8; 32], [8u8; 32], 3); + let (second, second_body) = sample_store_object([9u8; 32], [8u8; 32], 4); + store.save_proposal(&first, &first_body).await.unwrap(); + store.save_proposal(&second, &second_body).await.unwrap(); + let found = store.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, second.proposal_id); + let found = store.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); + assert_eq!(found[0].proposal_id, second.proposal_id); + } + + fn test_node(seed: u8) -> NodeId { + let mut raw = [0u8; 38]; + raw[..6].copy_from_slice(&[0x00, 0x24, 0x08, 0x01, 0x12, 0x20]); + raw[6..].fill(seed); + NodeId::from_raw(raw).unwrap() + } + + #[derive(Clone, Copy, Debug)] + enum PeerFault { + Timeout, + NotAValidator(NodeId), + Closed, + } + + impl PeerFault { + fn to_error(self) -> crate::realm::network::NetworkError { + use crate::realm::network::NetworkError; + match self { + PeerFault::Timeout => NetworkError::Timeout("test fault".to_string()), + PeerFault::NotAValidator(node) => NetworkError::NotAValidator(node), + PeerFault::Closed => NetworkError::CommandChannelClosed, + } + } + } + + #[derive(Default)] + struct PeerDoubleState { + offers: std::collections::HashMap>, + bodies: std::collections::HashMap<(NodeId, [u8; 32]), Result, PeerFault>>, + faults: std::collections::HashMap, + looked_up: tokio::sync::Mutex>, + inflight: std::sync::atomic::AtomicUsize, + max_inflight: std::sync::atomic::AtomicUsize, + } + + impl PeerDoubleState { + fn offer(mut self, peer: NodeId, lookup: RealmTransition, proposal: Proposal) -> Self { + self.offers.entry(peer).or_default().push((lookup, proposal)); + self + } + + fn body(mut self, peer: NodeId, proposal_id: [u8; 32], body: Result, PeerFault>) -> Self { + self.bodies.insert((peer, proposal_id), body); + self + } + + fn failing(mut self, peer: NodeId, fault: PeerFault) -> Self { + self.faults.insert(peer, fault); + self + } + } + + /// A command channel answering lookups and body ranges from a fixed inventory. + fn spawn_peer_double(state: std::sync::Arc) -> RealmNetworkCommands { + use crate::realm::network::{NetworkError, RealmNetworkCommand}; + let (commands, mut rx) = tokio::sync::mpsc::channel(64); + tokio::spawn(async move { + while let Some(command) = rx.recv().await { + match command { + RealmNetworkCommand::LookupProposal { + destination, + request, + response, + } => { + let live = state + .inflight + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + + 1; + state + .max_inflight + .fetch_max(live, std::sync::atomic::Ordering::SeqCst); + let answered = match state.faults.get(&destination) { + Some(fault) => Err(fault.to_error()), + None => { + let mut looked = state.looked_up.lock().await; + let offers = state.offers.get(&destination); + let mut entries = Vec::with_capacity(request.pairs.len()); + for pair in &request.pairs { + looked.insert((pair.old_root, pair.new_root)); + let candidates = offers + .map(|offers| { + offers + .iter() + .filter(|(lookup, _)| *lookup == *pair) + .map(|(_, proposal)| proposal.clone()) + .take(PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR) + .collect() + }) + .unwrap_or_default(); + entries.push(ProposalLookupEntry { + transition: *pair, + candidates, + }); + } + Ok(ProposalLookupResponse::candidates(entries)) + } + }; + let _ = response.send(answered); + state + .inflight + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + RealmNetworkCommand::RequestBody { + destination, + request, + response, + } => { + let answered = match state.bodies.get(&(destination, request.proposal_id)) { + Some(Ok(body)) => { + let start = request.offset as usize; + let take = (request.max_bytes as usize) + .min(body.len().saturating_sub(start)); + Ok(BodyChunkResponse { + offset: request.offset, + data: body[start..start + take].to_vec(), + eof: start + take == body.len(), + body_len: body.len() as u64, + body_hash: sha256(body), + }) + } + Some(Err(fault)) => Err(fault.to_error()), + None => Err(NetworkError::CommandChannelClosed), + }; + let _ = response.send(answered); + } + _ => {} + } + } + }); + RealmNetworkCommands::from_channel(commands, test_node(200)) + } + + fn lookup_of(old_root: [u8; 32], new_root: [u8; 32]) -> RealmTransition { + RealmTransition { old_root, new_root } + } + + fn single_peer(seed: u8) -> (Vec<(u16, NodeId)>, NodeId) { + let peer = test_node(seed); + (vec![(1, peer)], peer) + } + + async fn promote_all(store: &ProposalStore, outcomes: Vec) -> usize { + let mut promoted = 0usize; + for outcome in outcomes { + if let TransitionFetchOutcome::Staged(_, staged) = outcome { + store.install(staged).await.unwrap(); + promoted += 1; + } + } + promoted + } + + #[tokio::test] + async fn history_window_stages_each_offered_pair() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (first, first_body) = sample_store_object([1u8; 32], [2u8; 32], 1); + let (second, second_body) = sample_store_object([2u8; 32], [3u8; 32], 2); + let first_lookup = lookup_of([1u8; 32], [2u8; 32]); + let second_lookup = lookup_of([2u8; 32], [3u8; 32]); + let (members, peer) = single_peer(1); + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(peer, first_lookup, first.clone()) + .offer(peer, second_lookup, second.clone()) + .body(peer, first.proposal_id, Ok(first_body)) + .body(peer, second.proposal_id, Ok(second_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &store, + &peers, + 1, + 0, + &[first_lookup, second_lookup], + &[], + ) + .await; + assert_eq!(outcomes.len(), 2); + assert!(store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap().is_empty()); + assert_eq!(promote_all(&store, outcomes).await, 2); + assert_eq!( + store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap()[0].proposal_id, + first.proposal_id + ); + assert_eq!( + store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap()[0].proposal_id, + second.proposal_id + ); + } + + #[tokio::test] + async fn history_window_pair_failure_does_not_poison_sibling() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (good, good_body) = sample_store_object([4u8; 32], [5u8; 32], 5); + let (bad, mut bad_body) = sample_store_object([5u8; 32], [6u8; 32], 6); + bad_body[0] ^= 0xFF; + let good_lookup = lookup_of([4u8; 32], [5u8; 32]); + let bad_lookup = lookup_of([5u8; 32], [6u8; 32]); + let (members, peer) = single_peer(3); + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(peer, good_lookup, good.clone()) + .offer(peer, bad_lookup, bad.clone()) + .body(peer, good.proposal_id, Ok(good_body)) + .body(peer, bad.proposal_id, Ok(bad_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &store, + &peers, + 1, + 0, + &[good_lookup, bad_lookup], + &[], + ) + .await; + assert!(matches!(outcomes[0], TransitionFetchOutcome::Staged(..))); + assert!(matches!(outcomes[1], TransitionFetchOutcome::Failed(..))); + assert_eq!(promote_all(&store, outcomes).await, 1); + assert_eq!(store.lookup_transition(&[4u8; 32], &[5u8; 32]).await.unwrap().len(), 1); + assert!(store.lookup_transition(&[5u8; 32], &[6u8; 32]).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn history_lookup_failure_switches_to_backup_peer() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (proposal, body) = sample_store_object([7u8; 32], [8u8; 32], 7); + let lookup = lookup_of([7u8; 32], [8u8; 32]); + let primary = test_node(4); + let backup = test_node(5); + let members = vec![(1, primary), (2, backup)]; + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .failing(primary, PeerFault::Timeout) + .offer(backup, lookup, proposal.clone()) + .body(backup, proposal.proposal_id, Ok(body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &[lookup], &[]).await; + assert_eq!(promote_all(&store, outcomes).await, 1); + assert_eq!( + store.lookup_transition(&[7u8; 32], &[8u8; 32]).await.unwrap()[0].proposal_id, + proposal.proposal_id + ); + } + + #[tokio::test] + async fn history_window_fetch_concurrency_and_coverage() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (members, peer) = single_peer(8); + let mut state = PeerDoubleState::default(); + let mut needed = Vec::new(); + for i in 0..5u8 { + let old = [i; 32]; + let new = [i + 1; 32]; + let (proposal, mut body) = sample_store_object(old, new, 20 + i); + if i == 2 { + body[0] ^= 0xFF; + } + let lookup = lookup_of(old, new); + state = state + .offer(peer, lookup, proposal.clone()) + .body(peer, proposal.proposal_id, Ok(body)); + needed.push(lookup); + } + let state = std::sync::Arc::new(state); + let client = spawn_peer_double(state.clone()); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &needed, &[]).await; + assert_eq!(promote_all(&store, outcomes).await, 4); + assert!( + state.max_inflight.load(std::sync::atomic::Ordering::SeqCst) <= PROPOSAL_LOOKUP_CONCURRENCY + ); + let looked = state.looked_up.lock().await; + for lookup in &needed { + assert!( + looked.contains(&(lookup.old_root, lookup.new_root)), + "missing lookup pair=({},{})", + hex::encode(lookup.old_root), + hex::encode(lookup.new_root) + ); + } + drop(looked); + assert_eq!(store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap().len(), 0); + assert_eq!(store.lookup_transition(&[0u8; 32], &[1u8; 32]).await.unwrap().len(), 1); + assert_eq!(store.lookup_transition(&[3u8; 32], &[4u8; 32]).await.unwrap().len(), 1); + } + + #[tokio::test] + async fn history_window_across_epochs() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let epoch = CHECKPOINTS_PER_EPOCH; + let first_proposal_count = epoch - 1; + let second_proposal_count = epoch * 3 - 1; + let (first, first_body) = + sample_store_object_at([1u8; 32], [2u8; 32], 30, first_proposal_count, 1); + let (second, second_body) = + sample_store_object_at([2u8; 32], [3u8; 32], 31, second_proposal_count, 2); + let first_lookup = lookup_of([1u8; 32], [2u8; 32]); + let second_lookup = lookup_of([2u8; 32], [3u8; 32]); + let supplier = test_node(9); + let members = vec![(3, supplier)]; + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(supplier, first_lookup, first.clone()) + .offer(supplier, second_lookup, second.clone()) + .body(supplier, first.proposal_id, Ok(first_body)) + .body(supplier, second.proposal_id, Ok(second_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &store, + &peers, + 1, + 0, + &[first_lookup, second_lookup], + &[], + ) + .await; + assert_eq!(promote_all(&store, outcomes).await, 2); + assert_eq!(first.proposer_sub_id, 1); + assert_eq!(second.proposer_sub_id, 2); + let first_found = store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); + let second_found = store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap(); + assert_eq!(first_found[0].proposal_id, first.proposal_id); + assert_eq!(second_found[0].proposal_id, second.proposal_id); + assert!(first.base_checkpoint_id < second.base_checkpoint_id); + } + + #[tokio::test] + async fn history_window_without_anchor_leaf_fails_closed() { + let error = CatchupPeers::select(&[], 1).expect_err("empty occupied leaves must fail closed"); + assert!( + error + .to_string() + .contains("no other validator peer at this checkpoint"), + "{error}" + ); + } + + #[tokio::test] + async fn history_empty_answer_marks_every_pair_absent() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (members, _peer) = single_peer(10); + let state = std::sync::Arc::new(PeerDoubleState::default()); + let client = spawn_peer_double(state.clone()); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let needed = vec![lookup_of([8u8; 32], [9u8; 32])]; + let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &needed, &[]).await; + assert!(matches!(outcomes[0], TransitionFetchOutcome::Absent(..))); + assert!(state.looked_up.lock().await.contains(&([8u8; 32], [9u8; 32]))); + } + + #[tokio::test] + async fn history_resend_revotes() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (proposal, body) = sample_store_object([3u8; 32], [4u8; 32], 3); + store.save_proposal(&proposal, &body).await.unwrap(); + store.save_proposal(&proposal, &body).await.unwrap(); + let first = store + .read_body_chunk(&BodyChunkRequest { + proposal_id: proposal.proposal_id, + offset: 0, + max_bytes: 64, + }) + .await + .unwrap(); + store.save_proposal(&proposal, &body).await.unwrap(); + let second = store + .read_body_chunk(&BodyChunkRequest { + proposal_id: proposal.proposal_id, + offset: 0, + max_bytes: 64, + }) + .await + .unwrap(); + assert_eq!(first.body_hash, proposal.body_hash); + assert_eq!(second.body_hash, proposal.body_hash); + assert_eq!(first.body_len, second.body_len); + } + + #[tokio::test] + async fn history_invalid_candidate_ab() { + let dir = tempfile::tempdir().unwrap(); + let store = ProposalStore::open(dir.path()).await.unwrap(); + let (candidate_b, body_b) = sample_store_object([10u8; 32], [11u8; 32], 40); + let (mut candidate_x, body_x) = sample_store_object([10u8; 32], [12u8; 32], 41); + candidate_x.base_checkpoint_id = candidate_b.base_checkpoint_id; + store.save_proposal(&candidate_b, &body_b).await.unwrap(); + store.save_proposal(&candidate_x, &body_x).await.unwrap(); + let found = store.lookup_transition(&[10u8; 32], &[12u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, candidate_x.proposal_id); + let sibling = store.lookup_transition(&[10u8; 32], &[11u8; 32]).await.unwrap(); + assert_eq!(sibling[0].proposal_id, candidate_b.proposal_id); + let absent = store.lookup_transition(&[11u8; 32], &[10u8; 32]).await.unwrap(); + assert!(absent.is_empty()); + assert!(dir + .path() + .join("bodies") + .join(format!("{}_{}", hex::encode([10u8; 32]), hex::encode([11u8; 32]))) + .exists()); + } +} diff --git a/psy_node_core/src/config/node_start_config.rs b/psy_node_core/src/config/node_start_config.rs index fd077c59..b4c80689 100644 --- a/psy_node_core/src/config/node_start_config.rs +++ b/psy_node_core/src/config/node_start_config.rs @@ -46,6 +46,12 @@ impl RealmProcessorStartConfig { self.checkpoint_backup_path, self.realm_id, self.realm_sub_id ) } + pub fn get_proposal_backups_path(&self) -> String { + format!( + "{}/realm_{}_{}/proposal_backups", + self.checkpoint_backup_path, self.realm_id, self.realm_sub_id + ) + } } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/psy_node_core/src/p2p/traits/realm_coordinantor.rs b/psy_node_core/src/p2p/traits/realm_coordinantor.rs index 3dbc2192..ddfe93ca 100644 --- a/psy_node_core/src/p2p/traits/realm_coordinantor.rs +++ b/psy_node_core/src/p2p/traits/realm_coordinantor.rs @@ -13,7 +13,7 @@ pub trait RealmCoordinatorClient { async fn rc_get_checkpoint_leaves_batch(&self, start_checkpoint_id: u64, count: u32) -> anyhow::Result>; async fn rc_get_checkpoint_tree_merkle_proof(&self, checkpoint_id: u64) -> anyhow::Result>; async fn rc_get_realm_root_and_last_modified_checkpoint(&self, checkpoint_id: u64, realm_id: u64) -> anyhow::Result>; - async fn rc_submit_guta_proof(&self, input: GlobalUserTreeAggregatorHeaderWithTagValueAndJobType, proof: Vec, realm_id: u64, proposal: Option>, certificate: Option>, finalize_binding: Vec) -> anyhow::Result<()>; + async fn rc_submit_guta_proof(&self, input: GlobalUserTreeAggregatorHeaderWithTagValueAndJobType, proof: Vec, realm_id: u64, proposal: Option>, certificate: Option>) -> anyhow::Result<()>; async fn rc_get_contract_tree_state_heights(&self, checkpoint_id: u64, contract_ids: Vec) -> anyhow::Result>; } diff --git a/psy_node_core/src/p2p/validator_lookup.rs b/psy_node_core/src/p2p/validator_lookup.rs index e7175ac4..2578f129 100644 --- a/psy_node_core/src/p2p/validator_lookup.rs +++ b/psy_node_core/src/p2p/validator_lookup.rs @@ -1,10 +1,14 @@ use parth_core::{ crypto::hash::traits::MerkleZeroHasher, + data::hash::merkle_node_key::SimpleMerkleNodeKey, protocol::core_types::Q256BitHash, }; -use psy_data::p2p::{ - authenticate_validator_preimage, realm_validator_indexes, require_realm_validator_count, - BlsPublicKey, ValidatorLeafPreimage, +use psy_data::{ + guta::realm_finalize::VALIDATOR_TREE_HEIGHT, + p2p::{ + authenticate_validator_preimage, realm_validator_indexes, require_realm_validator_count, + BlsPublicKey, ValidatorLeaf, ValidatorLeafPreimage, + }, }; use crate::psy_core_db::traits::full::{ @@ -17,7 +21,12 @@ pub async fn load_realm_validators_from_tree( checkpoint_id: u64, realm_id: u32, expected_root: &Hash, -) -> anyhow::Result<(Vec, Vec<(u16, BlsPublicKey)>, Vec<(u16, u64)>)> +) -> anyhow::Result<( + Vec, + Vec<(u16, BlsPublicKey)>, + Vec<(u16, u64)>, + Vec<(u16, ValidatorLeaf)>, +)> where Hasher: MerkleZeroHasher, Hash: Copy + PartialEq + Q256BitHash, @@ -30,40 +39,74 @@ where ); let empty_leaf = Hasher::get_zero_hash(0); - let mut validator_sub_ids = Vec::new(); - let mut keys = Vec::new(); - let mut user_ids = Vec::new(); - for (sub_id, leaf_index) in realm_validator_indexes(realm_id) { - let leaf_hash = store - .validator_tree_get_leaf_hash(checkpoint_id, leaf_index) - .await?; - if leaf_hash == empty_leaf { - continue; - } - let preimage = store - .validator_tree_get_leaf_preimage(checkpoint_id, leaf_index) + let slots: Vec<(u16, u64)> = realm_validator_indexes(realm_id).collect(); + let leaf_keys: Vec = slots + .iter() + .map(|(_, leaf_index)| SimpleMerkleNodeKey::new(VALIDATOR_TREE_HEIGHT as u8, *leaf_index)) + .collect(); + let leaf_hashes = store.validator_tree_get_nodes(checkpoint_id, &leaf_keys).await?; + anyhow::ensure!( + leaf_hashes.len() == slots.len(), + "validator tree leaf batch returned {} hashes for {} realm {realm_id} slots", + leaf_hashes.len(), + slots.len() + ); + + let occupied: Vec<(u16, u64, Hash)> = slots + .iter() + .zip(leaf_hashes.iter()) + .filter(|(_, leaf_hash)| **leaf_hash != empty_leaf) + .map(|((sub_id, leaf_index), leaf_hash)| (*sub_id, *leaf_index, *leaf_hash)) + .collect(); + let preimage_indexes: Vec = occupied.iter().map(|(_, leaf_index, _)| *leaf_index).collect(); + let preimages = if preimage_indexes.is_empty() { + Vec::new() + } else { + store + .validator_tree_get_leaf_preimages(checkpoint_id, &preimage_indexes) .await? - .ok_or_else(|| { - anyhow::anyhow!( - "validator leaf preimage missing at realm {realm_id} sub {sub_id} checkpoint {checkpoint_id}" - ) - })?; + }; + anyhow::ensure!( + preimages.len() == occupied.len(), + "validator tree preimage batch returned {} values for {} occupied realm {realm_id} slots", + preimages.len(), + occupied.len() + ); + + let mut validator_sub_ids = Vec::with_capacity(occupied.len()); + let mut keys = Vec::with_capacity(occupied.len()); + let mut user_ids = Vec::with_capacity(occupied.len()); + let mut leaves = Vec::with_capacity(occupied.len()); + for ((sub_id, _leaf_index, leaf_hash), preimage) in occupied.iter().zip(preimages) { + let preimage = preimage.ok_or_else(|| { + anyhow::anyhow!( + "validator leaf preimage missing at realm {realm_id} sub {sub_id} checkpoint {checkpoint_id}" + ) + })?; anyhow::ensure!( preimage.chain_id == chain_id, "validator preimage chain_id {} does not match configured chain {chain_id} at realm {realm_id} sub {sub_id}", preimage.chain_id ); anyhow::ensure!( - preimage.realm_id == realm_id && preimage.realm_sub_id == sub_id, + preimage.realm_id == realm_id && preimage.realm_sub_id == *sub_id, "validator preimage slot mismatch at realm {realm_id} sub {sub_id}" ); - let leaf = authenticate_validator_preimage(&preimage, &leaf_hash)?; - validator_sub_ids.push(sub_id); - keys.push((sub_id, leaf.bls_public_key)); - user_ids.push((sub_id, preimage.validator_user_id)); + let leaf = authenticate_validator_preimage(&preimage, leaf_hash)?; + validator_sub_ids.push(*sub_id); + keys.push((*sub_id, leaf.bls_public_key)); + user_ids.push((*sub_id, preimage.validator_user_id)); + leaves.push((*sub_id, leaf)); } require_realm_validator_count(realm_id, validator_sub_ids.len())?; - Ok((validator_sub_ids, keys, user_ids)) + Ok((validator_sub_ids, keys, user_ids, leaves)) +} + +pub fn validator_nodes_from_leaves(leaves: &[(u16, ValidatorLeaf)]) -> Vec<(u16, psy_data::p2p::NodeId)> { + leaves + .iter() + .map(|(sub_id, leaf)| (*sub_id, leaf.node_id)) + .collect() } pub async fn write_validator_tree_genesis( @@ -84,3 +127,204 @@ where } Ok(()) } + + +#[cfg(test)] +mod tests { + use super::*; + use std::{collections::HashMap, sync::atomic::{AtomicUsize, Ordering}}; + use async_trait::async_trait; + use parth_core::{crypto::hash::merkle_proof::MerkleProofCore, pgoldilocks::PoseidonHasher, PHash}; + use psy_core::constants::chain_id::PSY_CHAIN_ID_LOCAL_DEVNET; + use psy_data::p2p::{BlsSecretKey, NodeId, NODE_ID_RAW_LEN}; + + const CHAIN: u64 = PSY_CHAIN_ID_LOCAL_DEVNET; + const REALM: u32 = 1; + const CP: u64 = 7; + + fn sample(sub: u16, user: u64, seed: u8) -> ValidatorLeafPreimage { + for i in 0u8..=32 { + let mut raw = [0u8; NODE_ID_RAW_LEN]; + raw[..6].copy_from_slice(&[0x00, 0x24, 0x08, 0x01, 0x12, 0x20]); + raw[6..].fill(seed.wrapping_add(i)); + let p = ValidatorLeafPreimage { + chain_id: CHAIN, realm_id: REALM, realm_sub_id: sub, validator_user_id: user, + node_id: *NodeId::from_raw(raw).unwrap().as_raw(), + bls_public_key: BlsSecretKey::key_gen(&[seed.wrapping_add(i); 32]).unwrap().public_key().to_bytes(), + }; + if p.leaf_hash().is_ok() { return p; } + } + panic!("no canonical preimage"); + } + fn h(p: &ValidatorLeafPreimage) -> PHash { PHash::from_owned_32bytes(p.leaf_hash().unwrap()) } + fn idx(sub: u16) -> u64 { ((REALM as u64) << 8) | sub as u64 } + + struct Tree { + roots: HashMap, + leaves: HashMap<(u64, u64), PHash>, + preimages: HashMap<(u64, u64), ValidatorLeafPreimage>, + leaf_q: AtomicUsize, pre_q: AtomicUsize, leaf_b: AtomicUsize, pre_b: AtomicUsize, + short_leaves: bool, short_preimages: bool, + } + impl Tree { + fn with(checkpoint: u64, root: PHash, occupied: &[ValidatorLeafPreimage]) -> Self { + let mut t = Self { + roots: HashMap::from([(checkpoint, root)]), + leaves: HashMap::new(), preimages: HashMap::new(), + leaf_q: AtomicUsize::new(0), pre_q: AtomicUsize::new(0), + leaf_b: AtomicUsize::new(0), pre_b: AtomicUsize::new(0), + short_leaves: false, short_preimages: false, + }; + for p in occupied { + t.occupy(checkpoint, p.clone()); + } + t + } + fn occupy(&mut self, checkpoint: u64, p: ValidatorLeafPreimage) { + let i = idx(p.realm_sub_id); + self.leaves.insert((checkpoint, i), h(&p)); + self.preimages.insert((checkpoint, i), p); + } + fn occupy_leaf(&mut self, checkpoint: u64, sub: u16, hash: PHash) { + self.leaves.insert((checkpoint, idx(sub)), hash); + } + fn latest_leaf(&self, checkpoint: u64, i: u64) -> PHash { + self.leaves.iter() + .filter(|((c, idx), _)| *idx == i && *c <= checkpoint) + .max_by_key(|((c, _), _)| *c) + .map(|(_, hash)| *hash) + .unwrap_or_else(|| PoseidonHasher::get_zero_hash(0)) + } + fn latest_preimage(&self, checkpoint: u64, i: u64) -> Option { + self.preimages.iter() + .filter(|((c, idx), _)| *idx == i && *c <= checkpoint) + .max_by_key(|((c, _), _)| *c) + .map(|(_, p)| p.clone()) + } + fn latest_root(&self, checkpoint: u64) -> PHash { + self.roots.iter() + .filter(|(c, _)| **c <= checkpoint) + .max_by_key(|(c, _)| **c) + .map(|(_, root)| *root) + .unwrap_or(self.roots[&CP]) + } + } + #[async_trait] + impl PsyNodeValidatorTreeDatabaseReader for Tree { + async fn validator_tree_get_leaf_hash(&self, c: u64, i: u64) -> anyhow::Result { + self.leaf_q.fetch_add(1, Ordering::SeqCst); + Ok(self.latest_leaf(c, i)) + } + async fn validator_tree_get_root_hash(&self, c: u64) -> anyhow::Result { Ok(self.latest_root(c)) } + async fn validator_tree_get_merkle_proof(&self, _c: u64, _i: u64) -> anyhow::Result> { unreachable!() } + async fn validator_tree_get_nodes(&self, c: u64, keys: &[SimpleMerkleNodeKey]) -> anyhow::Result> { + self.leaf_b.fetch_add(1, Ordering::SeqCst); + self.leaf_q.fetch_add(keys.len(), Ordering::SeqCst); + let mut out: Vec<_> = keys.iter().map(|k| self.latest_leaf(c, k.index)).collect(); + if self.short_leaves { out.pop(); } + Ok(out) + } + async fn validator_tree_get_node(&self, _c: u64, _k: SimpleMerkleNodeKey) -> anyhow::Result { unreachable!() } + async fn validator_tree_get_leaf_preimage(&self, c: u64, i: u64) -> anyhow::Result> { + self.pre_q.fetch_add(1, Ordering::SeqCst); + Ok(self.latest_preimage(c, i)) + } + async fn validator_tree_get_leaf_preimages(&self, c: u64, idxs: &[u64]) -> anyhow::Result>> { + self.pre_b.fetch_add(1, Ordering::SeqCst); + self.pre_q.fetch_add(idxs.len(), Ordering::SeqCst); + let mut out: Vec<_> = idxs.iter().map(|i| self.latest_preimage(c, *i)).collect(); + if self.short_preimages { out.pop(); } + Ok(out) + } + } + async fn load_at(t: &Tree, checkpoint: u64, root: &PHash) -> anyhow::Result<(Vec, Vec<(u16, BlsPublicKey)>, Vec<(u16, u64)>, Vec<(u16, ValidatorLeaf)>)> { + load_realm_validators_from_tree::(t, CHAIN, checkpoint, REALM, root).await + } + async fn load(t: &Tree, root: &PHash) -> anyhow::Result<(Vec, Vec<(u16, BlsPublicKey)>, Vec<(u16, u64)>, Vec<(u16, ValidatorLeaf)>)> { + load_at(t, CP, root).await + } + fn rejects(err: anyhow::Error, needle: &str) { + let text = err.to_string(); + assert!(text.contains(needle), "expected {needle:?} in {text:?}"); + } + + #[tokio::test] + async fn two_validators_authenticate_query_count_and_empty_slots() { + let a = sample(1, 11, 11); let b = sample(5, 15, 15); let root = h(&a); + let t = Tree::with(CP, root, &[a.clone(), b.clone()]); + let empty = PoseidonHasher::get_zero_hash(0); + for (_, i) in realm_validator_indexes(REALM) { + if t.validator_tree_get_leaf_hash(CP, i).await.unwrap() != empty { + t.validator_tree_get_leaf_preimage(CP, i).await.unwrap(); + } + } + assert_eq!((t.leaf_q.load(Ordering::SeqCst), t.pre_q.load(Ordering::SeqCst)), (256, 2)); + t.leaf_q.store(0, Ordering::SeqCst); t.pre_q.store(0, Ordering::SeqCst); + let (subs, keys, users, leaves) = load(&t, &root).await.unwrap(); + assert_eq!(subs, vec![1, 5]); + assert_eq!(users, vec![(1, 11), (5, 15)]); + assert_eq!(keys[0].1, a.to_leaf().unwrap().bls_public_key); + assert_eq!(leaves[1].1, b.to_leaf().unwrap()); + assert_eq!((t.leaf_b.load(Ordering::SeqCst), t.pre_b.load(Ordering::SeqCst)), (1, 1)); + assert_eq!((t.leaf_q.load(Ordering::SeqCst), t.pre_q.load(Ordering::SeqCst)), (256, 2)); + } + + #[tokio::test] + async fn missing_occupied_preimage_rejects() { + let a = sample(1, 11, 11); let root = h(&a); + let mut t = Tree::with(CP, root, &[]); + t.occupy_leaf(CP, 1, root); + rejects(load(&t, &root).await.unwrap_err(), "preimage missing"); + } + + #[tokio::test] + async fn corrupted_preimage_rejects() { + let a = sample(1, 11, 11); let root = h(&a); + let mut t = Tree::with(CP, root, &[a]); + t.occupy_leaf(CP, 1, h(&sample(5, 15, 15))); + rejects(load(&t, &root).await.unwrap_err(), "hash mismatch"); + } + + #[tokio::test] + async fn historical_checkpoint_uses_old_leaf_and_preimage() { + let old = sample(1, 11, 11); let new = sample(1, 99, 21); + let old_root = h(&old); let new_root = h(&new); + let mut t = Tree::with(3, old_root, &[old]); + t.roots.insert(CP, new_root); + t.occupy(CP, new); + let users = load_at(&t, 3, &old_root).await.unwrap().2; + assert_eq!(users, vec![(1, 11)]); + let users = load_at(&t, CP, &new_root).await.unwrap().2; + assert_eq!(users, vec![(1, 99)]); + } + + #[tokio::test] + async fn empty_realm_hits_count_gate_without_preimage_batch() { + let empty_root = PoseidonHasher::get_zero_hash(VALIDATOR_TREE_HEIGHT); + let empty = Tree::with(CP, empty_root, &[]); + rejects(load(&empty, &empty_root).await.unwrap_err(), "validator count 0"); + assert_eq!(empty.pre_b.load(Ordering::SeqCst), 0); + assert_eq!(empty.pre_q.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn short_batches_reject() { + let a = sample(1, 11, 11); let b = sample(5, 15, 15); let root = h(&a); + let mut t = Tree::with(CP, root, &[a.clone(), b]); + t.short_leaves = true; + rejects(load(&t, &root).await.unwrap_err(), "leaf batch returned"); + t.short_leaves = false; t.short_preimages = true; + rejects(load(&t, &root).await.unwrap_err(), "preimage batch returned"); + } + + #[tokio::test] + async fn invalid_bls_rejects() { + let a = sample(1, 11, 11); + let mut bad = a; bad.bls_public_key = [0u8; 48]; + let dummy = PHash::from_owned_32bytes([1u8; 32]); + let mut t = Tree::with(CP, dummy, &[]); + t.occupy_leaf(CP, 1, dummy); + t.preimages.insert((CP, idx(1)), bad); + rejects(load(&t, &dummy).await.unwrap_err(), "invalid stored validator BLS key"); + } +} diff --git a/psy_node_core/src/psy_core_db/core_implementation/base.rs b/psy_node_core/src/psy_core_db/core_implementation/base.rs index 9798de98..d6e4df53 100644 --- a/psy_node_core/src/psy_core_db/core_implementation/base.rs +++ b/psy_node_core/src/psy_core_db/core_implementation/base.rs @@ -10,7 +10,7 @@ use parth_core::{ merkle_store_key::{QMerkleStoreDoubleIdNode, QMerkleStoreSingleIdNode}, }, felt::ToU64Value, - protocol::core_types::QNetworkDatabaseTypes, + protocol::core_types::{Q256BitHash, QNetworkDatabaseTypes}, QCoreProcCheckpointUniqueId, }; use psy_data::v1::qdata::{ diff --git a/psy_node_core/src/psy_core_db/core_implementation/constants.rs b/psy_node_core/src/psy_core_db/core_implementation/constants.rs index 6c2206ff..815b5d12 100644 --- a/psy_node_core/src/psy_core_db/core_implementation/constants.rs +++ b/psy_node_core/src/psy_core_db/core_implementation/constants.rs @@ -1,5 +1,6 @@ pub const U64_SINGLETON_TABLE_OBJ_ID_CHECKPOINT_ID: u64 = 1; pub const U64_SINGLETON_TABLE_OBJ_ID_PENDING_ID: u64 = 2; +pub const U64_SINGLETON_TABLE_OBJ_ID_GENESIS_COMPLETE: u64 = 3; pub const LATEST_INFO_TABLE_OBJ_ID_LATEST_L2_BLOCK_STATE: u64 = 1; pub const LATEST_INFO_TABLE_OBJ_ID_LATEST_CHECKPOINT_TREE_ROOT: u64 = 2; diff --git a/psy_node_core/src/psy_core_db/traits/full.rs b/psy_node_core/src/psy_core_db/traits/full.rs index b6df5817..fdb415fa 100644 --- a/psy_node_core/src/psy_core_db/traits/full.rs +++ b/psy_node_core/src/psy_core_db/traits/full.rs @@ -63,6 +63,7 @@ pub trait PsyNodeValidatorTreeDatabaseReader { async fn validator_tree_get_nodes(&self, checkpoint_id: u64, keys: &[SimpleMerkleNodeKey]) -> anyhow::Result>; async fn validator_tree_get_node(&self, checkpoint_id: u64, key: SimpleMerkleNodeKey) -> anyhow::Result; async fn validator_tree_get_leaf_preimage(&self, checkpoint_id: u64, leaf_index: u64) -> anyhow::Result>; + async fn validator_tree_get_leaf_preimages(&self, checkpoint_id: u64, leaf_indexes: &[u64]) -> anyhow::Result>>; } #[async_trait] @@ -90,6 +91,13 @@ pub trait PsyNodeGlobalUserTreeDatabaseReader { async fn global_user_tree_get_nodes(&self, checkpoint_id: u64, keys: &[SimpleMerkleNodeKey]) -> anyhow::Result>; async fn global_user_tree_get_node(&self, checkpoint_id: u64, key: SimpleMerkleNodeKey) -> anyhow::Result; async fn global_user_tree_dump_all_leaves(&self, checkpoint_id: u64) -> anyhow::Result>; + async fn global_user_tree_dump_leaves_range( + &self, + checkpoint_id: u64, + min_user_id_inclusive: u64, + max_user_id_exclusive: u64, + ) -> anyhow::Result>; + async fn global_user_tree_get_node_and_checkpoint_id_max_checkpoint( &self, max_checkpoint_id: u64, @@ -275,6 +283,7 @@ pub trait PsyNodeContractFunctionTreeDatabaseWriter { #[auto_impl(&, Arc)] pub trait PsyNodeCheckpointObjectDatabaseReader { async fn get_latest_checkpoint_id(&self) -> anyhow::Result; + async fn get_genesis_complete(&self) -> anyhow::Result; async fn get_checkpoint_id_for_checkpoint_root_hash(&self, root_hash: Hash) -> anyhow::Result>; async fn get_checkpoint_leaf_data(&self, checkpoint_id: u64) -> anyhow::Result>; async fn get_l2_block_state(&self, checkpoint_id: u64) -> anyhow::Result; @@ -358,6 +367,7 @@ pub trait PsyNodeCheckpointObjectDatabaseWriter { unique_id_struct: &QCoreProcCheckpointUniqueId, ) -> anyhow::Result<()>; async fn set_latest_checkpoint_id(&self, checkpoint_id: u64) -> anyhow::Result<()>; + async fn set_genesis_complete(&self) -> anyhow::Result<()>; async fn set_checkpoint_leaf_data(&self, checkpoint_id: u64, leaf_data: &PQEDCheckpointLeaf) -> anyhow::Result<()>; async fn set_checkpoint_root_hash_to_id_mapping(&self, checkpoint_root: Hash, checkpoint_id: u64) -> anyhow::Result<()>; async fn set_l2_latest_block_state(&self, block_state: &QEDL2BlockState) -> anyhow::Result<()>; diff --git a/psy_node_core/src/psy_core_db/v3_implementation/full.rs b/psy_node_core/src/psy_core_db/v3_implementation/full.rs index 82a87bb1..2abf747f 100644 --- a/psy_node_core/src/psy_core_db/v3_implementation/full.rs +++ b/psy_node_core/src/psy_core_db/v3_implementation/full.rs @@ -21,7 +21,7 @@ use parth_core::{ serializable::QPDSerializable, }, felt::ToU64Value, - protocol::core_types::QNetworkDatabaseTypes, + protocol::core_types::{Q256BitHash, QNetworkDatabaseTypes}, QCoreProcCheckpointUniqueId, }; use psy_data::{ @@ -41,7 +41,8 @@ use crate::{ CHECKPOINTED_OBJECT_TABLE_OBJ_ID_REALM_ROOT_TO_GLOBAL_USER_TREE_ROOT_MERKLE_PROOF, CHECKPOINTED_OBJECT_TABLE_OBJ_ID_VALIDATOR_TREE_PREIMAGE_BASE, LATEST_INFO_TABLE_OBJ_ID_LATEST_L2_BLOCK_STATE, - U64_SINGLETON_TABLE_OBJ_ID_BRIDGE_DEPOSIT_NEXT_INDEX_BASE, U64_SINGLETON_TABLE_OBJ_ID_CHECKPOINT_ID, U64_SINGLETON_TABLE_OBJ_ID_PENDING_ID, + U64_SINGLETON_TABLE_OBJ_ID_BRIDGE_DEPOSIT_NEXT_INDEX_BASE, U64_SINGLETON_TABLE_OBJ_ID_CHECKPOINT_ID, + U64_SINGLETON_TABLE_OBJ_ID_GENESIS_COMPLETE, U64_SINGLETON_TABLE_OBJ_ID_PENDING_ID, }, traits::full::*, }, @@ -376,6 +377,13 @@ impl< None => Ok(0), } } + pub async fn get_genesis_complete(&self) -> anyhow::Result { + let v = self + .store + .db_select_u64_value(&self.u64_singleton_table, U64_SINGLETON_TABLE_OBJ_ID_GENESIS_COMPLETE) + .await?; + Ok(v == Some(1)) + } pub async fn get_latest_pending_id(&self) -> anyhow::Result { let v = self .store @@ -888,6 +896,24 @@ impl< ) .await } + + async fn validator_tree_get_leaf_preimages( + &self, + checkpoint_id: u64, + leaf_indexes: &[u64], + ) -> anyhow::Result>> { + let obj_ids = leaf_indexes + .iter() + .map(|leaf_index| validator_tree_preimage_obj_id(*leaf_index)) + .collect::>>()?; + self.store + .db_select_many_single_checkpointed_object_values::( + &self.checkpointed_object_table, + &obj_ids, + checkpoint_id, + ) + .await + } } #[async_trait] @@ -1101,6 +1127,25 @@ impl< .db_dump_all_zero_id_merkle_node_leaves_chunked(&self.global_user_tree_table, checkpoint_id) .await } + async fn global_user_tree_dump_leaves_range( + &self, + checkpoint_id: u64, + min_user_id_inclusive: u64, + max_user_id_exclusive: u64, + ) -> anyhow::Result> { + if min_user_id_inclusive >= max_user_id_exclusive { + return Ok(HashMap::new()); + } + self.store + .db_dump_zero_id_merkle_node_leaves_range( + &self.global_user_tree_table, + checkpoint_id, + min_user_id_inclusive, + max_user_id_exclusive - 1, + ) + .await + } + async fn global_user_tree_get_node_and_checkpoint_id_max_checkpoint( &self, @@ -2197,6 +2242,10 @@ impl< self.get_latest_checkpoint_id().await } + async fn get_genesis_complete(&self) -> anyhow::Result { + self.get_genesis_complete().await + } + async fn get_checkpoint_id_for_checkpoint_root_hash(&self, root_hash: N::QHash) -> anyhow::Result> { self.store .db_select_one_by_k1::(&self.checkpoint_root_to_checkpoint_id_table, &root_hash) @@ -2303,6 +2352,7 @@ impl< .db_select_u64_value(&self.pending_id_to_checkpoint_id_table, unique_pending_id) .await } + async fn get_proc_checkpoint_unique_id_for_pending_id(&self, unique_pending_id: u64) -> anyhow::Result> { self.store .db_select_one_u128_value_by_u64(&self.pending_id_to_pending_proc_id_table, unique_pending_id) @@ -2712,6 +2762,12 @@ impl< .await } + async fn set_genesis_complete(&self) -> anyhow::Result<()> { + self.store + .db_set_u64_value(&self.u64_singleton_table, U64_SINGLETON_TABLE_OBJ_ID_GENESIS_COMPLETE, 1) + .await + } + async fn set_checkpoint_leaf_data(&self, checkpoint_id: u64, leaf_data: &PQEDCheckpointLeaf) -> anyhow::Result<()> { self.store.db_insert_one_kiv(&self.checkpoint_leaf_table, checkpoint_id, leaf_data).await } diff --git a/psy_node_core/src/store/traits/core_db.rs b/psy_node_core/src/store/traits/core_db.rs index dae9972a..bd1915f2 100644 --- a/psy_node_core/src/store/traits/core_db.rs +++ b/psy_node_core/src/store/traits/core_db.rs @@ -600,6 +600,14 @@ pub trait CoreDatabaseZeroIdMerkleDumpReader< max_checkpoint_id: u64, strategy: MerkleTreeDumpStrategy, ) -> anyhow::Result>>; + async fn db_dump_zero_id_merkle_node_leaves_range( + &self, + table: &TableIdentifier, + max_checkpoint_id: u64, + start_index: u64, + end_index: u64, + ) -> anyhow::Result>; + /* async fn dump_all_zero_id_merkle_node_leaves_chunked< F: Send + Sync + FnMut(Vec<(u64, Hash)>) -> Fut, diff --git a/psy_node_scylla/src/core_db.rs b/psy_node_scylla/src/core_db.rs index 718baeca..cba326cc 100644 --- a/psy_node_scylla/src/core_db.rs +++ b/psy_node_scylla/src/core_db.rs @@ -503,6 +503,21 @@ impl + Send + Sy ) -> anyhow::Result>>{ table.dump_all_zero_id_merkle_node_leaves_vec::(&self.session, max_checkpoint_id, strategy).await } + + async fn db_dump_zero_id_merkle_node_leaves_range( + &self, + table: &ScyllaMerkleNodesZeroPreparedStatements, + max_checkpoint_id: u64, + start_index: u64, + end_index: u64, + ) -> anyhow::Result> { + if start_index > end_index { + return Ok(HashMap::new()); + } + table + .dump_leaves_stream_end_index::(&self.session, max_checkpoint_id, start_index, end_index) + .await + } } #[async_trait] diff --git a/psy_node_scylla/src/tables/merkle/zero.rs b/psy_node_scylla/src/tables/merkle/zero.rs index 17246c52..3e902c1c 100644 --- a/psy_node_scylla/src/tables/merkle/zero.rs +++ b/psy_node_scylla/src/tables/merkle/zero.rs @@ -507,8 +507,8 @@ impl ScyllaMerkleNodesZeroPreparedStatements { let hash = Hash::from_slice_32bytes(&value)?; output_map.insert(node_index, hash); + prev_index = Some(node_index_i64); } - prev_index = Some(node_index_i64); } // Else skip historical for same index } @@ -516,7 +516,7 @@ impl ScyllaMerkleNodesZeroPreparedStatements { } // Consolidated dump: stream leaf level, dedup client-side for latest <= // max_checkpoint - async fn dump_leaves_stream_end_index( + pub(crate) async fn dump_leaves_stream_end_index( &self, session: &Session, max_checkpoint_id: u64, @@ -543,8 +543,8 @@ impl ScyllaMerkleNodesZeroPreparedStatements { let hash = Hash::from_slice_32bytes(&value)?; output_map.insert(node_index, hash); + prev_index = Some(node_index_i64); } - prev_index = Some(node_index_i64); } // Else skip historical for same index } diff --git a/psy_node_scylla/tests/zero_id_dump_v2.rs b/psy_node_scylla/tests/zero_id_dump_v2.rs index 93ec9f49..a4bce8c3 100644 --- a/psy_node_scylla/tests/zero_id_dump_v2.rs +++ b/psy_node_scylla/tests/zero_id_dump_v2.rs @@ -6,7 +6,7 @@ use dashmap::DashMap; use parth_core::{crypto::hash::traits::MerkleZeroHasher, data::{db::table::QDatabaseTableRoutingKey, hash::{hash256::Hash256, merkle_node_key::{SimpleMerkleNode, SimpleMerkleNodeKey}}, serializable::QPDPair}, protocol::core_types::{QDBHashBase}}; use parth_crypto::hash::sha256::CoreSha256Hasher; use psy_node_scylla::{core::ScyllaCoreStore, tables::merkle::ScyllaMerkleNodesZeroPreparedStatements}; -use psy_node_core::store::traits::{core_db::{CoreDatabaseZeroIdMerkleDumpReader, CoreDatabaseZeroIdMerkleReader, CoreDatabaseZeroIdMerkleStore}, helpers::db_helper_zero_id_merkle_node_simple_set_leaves}; +use psy_node_core::store::traits::{core_db::{CoreDatabaseZeroIdMerkleDumpReader, CoreDatabaseZeroIdMerkleReader, CoreDatabaseZeroIdMerkleStore, CoreDatabaseZeroIdMerkleWriter, MerkleTreeDumpStrategy}, helpers::db_helper_zero_id_merkle_node_simple_set_leaves}; use serde::Serialize; @@ -423,4 +423,76 @@ async fn simple_store_basic_test_1() -> anyhow::Result<()> { println!("setup simple store"); simple_store.basic_test_1().await?; Ok(()) +} + +#[tokio::test] +#[ignore = "database slow"] +async fn historical_dump_matches_point_select_at_prior_checkpoint() -> anyhow::Result<()> { + let key_space = format!("psy_node_zero_id_dump_hist_{}", rand::random::()); + let scylla_db = ScyllaCoreStore::::new(0, 0, key_space, &[ + "127.0.0.1:9042".to_string() + ]).await?; + let session = scylla_db.session.clone(); + let keyspace = scylla_db.keyspace.clone(); + let result: anyhow::Result<()> = async { + let simple_store = SimpleStoreEx::setup(Arc::new(scylla_db)).await?; + let table = simple_store.store.merkle_node_zero_id_table_a.as_ref(); + let store = simple_store.store.store.as_ref(); + let level = EX_ZERO_ID_TREE_A_HEIGHT as u8; + let expected_100 = [ + Hash256([0xA1; 32]), + Hash256([0xA2; 32]), + Hash256([0xA3; 32]), + ]; + let expected_200 = [ + Hash256([0xB1; 32]), + Hash256([0xB2; 32]), + ExHasher::get_zero_hash(0), + ]; + store.db_set_zero_id_merkle_nodes_batch(table, 100, &[ + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 0 }, value: expected_100[0] }, + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 1 }, value: expected_100[1] }, + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 2 }, value: expected_100[2] }, + ]).await?; + store.db_set_zero_id_merkle_nodes_batch(table, 200, &[ + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 0 }, value: expected_200[0] }, + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 1 }, value: expected_200[1] }, + SimpleMerkleNode { key: SimpleMerkleNodeKey { level, index: 2 }, value: expected_200[2] }, + ]).await?; + for (checkpoint, expected_values) in [(100u64, expected_100), (200u64, expected_200)] { + let mut expected = HashMap::new(); + for index in 0..3u64 { + let key = SimpleMerkleNodeKey { level, index }; + let oracle = store.db_select_zero_id_merkle_node_max_checkpoint(table, checkpoint, &key).await?; + if oracle != expected_values[index as usize] { + return Err(anyhow::anyhow!("point select at cp{checkpoint} index {index} expected {:?} got {:?}", expected_values[index as usize], oracle)); + } + expected.insert(index, expected_values[index as usize]); + } + let full = store.db_dump_all_zero_id_merkle_node_leaves_vec(table, checkpoint, MerkleTreeDumpStrategy::DumpAllStrategy).await?; + let bounded = store.db_dump_all_zero_id_merkle_node_leaves_vec(table, checkpoint, MerkleTreeDumpStrategy::AppendOnlyTreeStrategy).await?; + let full_map: HashMap = full.into_iter().map(|n| (n.key.index, n.value)).collect(); + let bounded_map: HashMap = bounded.into_iter().map(|n| (n.key.index, n.value)).collect(); + if full_map != expected { + return Err(anyhow::anyhow!("full dump at cp{checkpoint} expected {:?} got {:?}", expected, full_map)); + } + if bounded_map != expected { + return Err(anyhow::anyhow!("bounded dump at cp{checkpoint} expected {:?} got {:?}", expected, bounded_map)); + } + } + Ok(()) + }.await; + let cleanup = async { + session.query_unpaged(format!("DROP KEYSPACE IF EXISTS {keyspace}"), &[]).await?; + session.await_schema_agreement().await?; + session.query_unpaged(format!("DROP KEYSPACE IF EXISTS {keyspace}_no_tablet"), &[]).await?; + session.await_schema_agreement().await?; + Ok::<(), anyhow::Error>(()) + }.await; + match (result, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(e), Ok(())) => Err(e), + (Ok(()), Err(c)) => Err(c), + (Err(e), Err(c)) => Err(anyhow::anyhow!("{e:#}; cleanup also failed: {c:#}")), + } } \ No newline at end of file diff --git a/psy_node_store_memory/src/v2/cbs_store.rs b/psy_node_store_memory/src/v2/cbs_store.rs index b3cab423..fc1768b0 100644 --- a/psy_node_store_memory/src/v2/cbs_store.rs +++ b/psy_node_store_memory/src/v2/cbs_store.rs @@ -1262,6 +1262,41 @@ where } + async fn db_dump_zero_id_merkle_node_leaves_range( + &self, + table: &InMemoryTableIdentifier, + max_checkpoint_id: u64, + start_index: u64, + end_index: u64, + ) -> anyhow::Result> { + if start_index > end_index { + return Ok(HashMap::new()); + } + let db = self.get_or_create_table(&table.to_string()); + let tree_height = table.tree_height; + let start_key = key_helpers::key_merkle_zero_id( + &SimpleMerkleNodeKey { level: tree_height, index: start_index }, + 0, + ); + let end_key = key_helpers::key_merkle_zero_id( + &SimpleMerkleNodeKey { level: tree_height, index: end_index }, + u64::MAX, + ); + let mut result_map = HashMap::new(); + for entry in db.range(start_key..=end_key) { + let key_bytes = entry.key(); + let index = u64::from_be_bytes(key_bytes[1..9].try_into()?); + if index < start_index || index > end_index { + continue; + } + let checkpoint_id = u64::from_be_bytes(key_bytes[9..17].try_into()?); + if checkpoint_id > max_checkpoint_id { + continue; + } + result_map.insert(index, Hash::from_bytes(entry.value())?); + } + Ok(result_map) + } } #[async_trait] impl CoreDatabaseZeroIdMerkleReader From 1cd9c5803280b1b3c52fae21f69c30a72f5876e0 Mon Sep 17 00:00:00 2001 From: 0x8f701 Date: Fri, 18 Sep 2026 16:52:47 +0800 Subject: [PATCH 2/3] Split FFS recovery and name derive-shield from the layout. - Split recovery into ffs/{layout,baseline_replay,verify,adopt}; tests follow owners. - Decode proposal FFS roots from the typed layout; drop STATE_UPDATES_ROOTS_OFFSET. - Replace derive-note-owner with derive-shield; accept private-key XOR user-id. - Expand TERMINOLOGY.md past Realm/P2P; distinguish build from derive. --- AGENTS.md | 60 +- .../psy_user_cli/examples/derive_shield.rs | 25 - .../psy_cli/psy_user_cli/src/main.rs | 8 +- .../psy_cli/psy_user_cli/src/result.rs | 32 +- .../psy_user_cli/src/subcommand/args.rs | 8 +- .../psy_user_cli/src/subcommand/mod.rs | 4 +- .../src/subcommand/shield_address.rs | 134 +- docs/src/dev/TERMINOLOGY.md | 235 ++ docs/src/dev/bridge-common-operations.md | 8 +- docs/src/dev/gatherers.md | 13 +- docs/src/dev/private-transfer.md | 20 +- docs/src/dev/processors.md | 15 +- docs/src/dev/realm-recovery-e2e.md | 84 +- docs/src/dev/rollback-operations.md | 4 +- docs/src/rpc/UserCli.md | 4 +- psy_cli/psy_node_cli/src/node/realm_p2p.rs | 14 +- .../src/node/startup_plonky2_scylla.rs | 8 +- .../src/node/startup_processor_jtmb_scylla.rs | 8 +- psy_data/src/p2p/messages.rs | 50 +- psy_data/src/p2p/mod.rs | 4 +- psy_node_common/src/realm/network/codec.rs | 14 +- .../src/realm/processor/catchup.rs | 476 +++- .../src/realm/processor/core/mod.rs | 4 +- .../src/realm/processor/core/process_block.rs | 52 +- .../src/realm/processor/core/runner.rs | 2 +- .../src/realm/processor/core/startup.rs | 24 +- psy_node_common/src/realm/processor/create.rs | 4 +- .../src/realm/processor/db/commit.rs | 6 +- .../src/realm/processor/db/genesis.rs | 2 + .../src/realm/processor/db/init.rs | 126 +- psy_node_common/src/realm/processor/db/mod.rs | 4 + .../src/realm/processor/db/records.rs | 511 ++++ .../src/realm/processor/ffs/adopt.rs | 106 + .../realm/processor/ffs/baseline_replay.rs | 424 +++ .../src/realm/processor/ffs/layout.rs | 405 +++ .../src/realm/processor/ffs/mod.rs | 167 ++ .../src/realm/processor/ffs/verify.rs | 583 ++++ psy_node_common/src/realm/processor/mod.rs | 4 +- .../{proposal_store.rs => proposal_backup.rs} | 189 +- .../src/realm/processor/recovery.rs | 2384 ----------------- 40 files changed, 3460 insertions(+), 2765 deletions(-) delete mode 100644 client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs create mode 100644 docs/src/dev/TERMINOLOGY.md create mode 100644 psy_node_common/src/realm/processor/db/records.rs create mode 100644 psy_node_common/src/realm/processor/ffs/adopt.rs create mode 100644 psy_node_common/src/realm/processor/ffs/baseline_replay.rs create mode 100644 psy_node_common/src/realm/processor/ffs/layout.rs create mode 100644 psy_node_common/src/realm/processor/ffs/mod.rs create mode 100644 psy_node_common/src/realm/processor/ffs/verify.rs rename psy_node_common/src/realm/processor/{proposal_store.rs => proposal_backup.rs} (71%) delete mode 100644 psy_node_common/src/realm/processor/recovery.rs diff --git a/AGENTS.md b/AGENTS.md index 05add0d8..13346a01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,10 @@ This file governs `psy-node` and coordinated changes across the sibling PsyProto It supplements higher-level agent rules. The stricter rule wins. +## Required Terminology Reading + +Before naming a function, variable, field, type, module, file, or log marker, every AI agent MUST read `docs/src/dev/TERMINOLOGY.md`. That file owns the shared verbs and domain nouns. Do not create a second term for a concept it already names; reuse the existing word. If a needed term is absent, add it to `docs/src/dev/TERMINOLOGY.md` in the same change that introduces the symbol. + ## Required Devnet Lifecycle Reading Before any devnet startup, shutdown, restart, rollback, or live E2E operation, every AI agent MUST read and follow `docs/src/dev/devnet_lifecycle.md` and `docs/src/dev/devnet-launcher-reference.md`. The lifecycle guide owns state-preserving operations; the launcher reference owns launcher flags, the startup DAG, ports, environment, Anvil persistence, and known current-source limitations. JTMB remains test-only and is not rollback-validation evidence. Rollback validation uses the Plonky2 path; all lifecycle stop/resume, artifact, and verification procedures remain authoritative. @@ -589,19 +593,20 @@ Required fields per note: ## Naming -1. Functions state what they do, variables state what they represent, and types state what they model. -2. Avoid vague names such as `tmp`, `data`, `result`, `obj`, and `foo` in long-lived or public interfaces. -3. Use only universally understood abbreviations such as `ctx`, `id`, `cfg`, `db`, and `tx`. -4. Use the same name for the same concept throughout the repository. -5. Do not rename an existing value when creating its target, constant, reference, witness, or borrowed form. Preserve the established concept name with a structural suffix only when the type requires distinction, for example `guta_circuit_whitelist_root` and `guta_circuit_whitelist_root_target`; subjective aliases such as `official_whitelist_root`, `canonical_root`, or `expected_root` for that same value are forbidden. -6. Prefix booleans with `is_`, `has_`, `should_`, or `can_`. -7. Do not embed task identifiers, phase numbers, or step numbers in code names, file names, comments, or commit messages. -8. Version numbers belong to the runtime revision field and migration manifests, not to type names. If a migration window forces two schemas to coexist, name both by role and remove the older schema in the next storage-layout revision. -9. Lifecycle and authority labels (`legacy`, `old`, `deprecated`, `retired`, `canonical`, `official`, `v1`, `v2`) never name code, files, or documentation. Keep exactly one optimal implementation. A proxy-upgrade storage mirror is named by what it holds (for example `ImportedTokenFlowConfig`) and is removed in the next storage-layout revision. -10. Prefer one common verb plus one concrete object for function names. Use direct verbs such as `get`, `set`, `check`, `read`, `write`, `load`, `save`, `add`, `remove`, `create`, `update`, `apply`, and `build`; avoid multi-verb names, process jargon, and subjective lifecycle labels when one plain action states the behavior. `persist` is banned: write durable data with `save`/`write`, read it back with `load`/`read`; do not introduce `persist*`, `store` as a verb synonym for those, or other interchangeable synonyms for the same operation. -11. Name quantities as `__`, with the unit last: `remaining_backup_bytes`, `max_proof_bytes`, `processed_user_count`. Avoid sentence fragments such as `backup_bytes_remaining` and implementation terms such as `allocation` when the value is a validated size or count. -12. Use common verbs consistently: `get` reads an existing value; `load` assembles a domain value from durable storage; `read` decodes a file or byte stream; `build` derives a value without persistence; `create` makes a new stored or runtime object; `set` replaces a whole value; `update` changes part of a value; `apply` executes a state transition; `validate` checks untrusted or serialized input; `ensure` enforces an internal invariant and returns an error; `check` returns a state classification or health result. Do not use these as interchangeable synonyms. -13. Order composite domain names from scope to object to representation: `checkpoint_tree_root`, `validator_tree_proof`, `user_leaf_hash`, `gathering_checkpoint_id`. State qualifiers precede the domain name (`current_`, `next_`, `last_committed_`, `remaining_`, `expected_`, `actual_`); collections use plural nouns and identifiers end in `_id` or `_ids`. +1. Read `docs/src/dev/TERMINOLOGY.md` before choosing any new function, variable, field, or type name. Reuse a term that file already defines. A second name for the same concept is forbidden. +2. Functions state what they do, variables state what they represent, and types state what they model. +3. Avoid vague names such as `tmp`, `data`, `result`, `obj`, and `foo` in long-lived or public interfaces. +4. Use only universally understood abbreviations such as `ctx`, `id`, `cfg`, `db`, and `tx`. +5. Use the same name for the same concept throughout the repository. Do not introduce a synonym, alias, or parallel term. +6. Do not rename an existing value when creating its target, constant, reference, witness, or borrowed form. Preserve the established concept name with a structural suffix only when the type requires distinction, for example `guta_circuit_whitelist_root` and `guta_circuit_whitelist_root_target`; subjective aliases such as `official_whitelist_root`, `canonical_root`, or `expected_root` for that same value are forbidden. +7. Prefix booleans with `is_`, `has_`, `should_`, or `can_`. +8. Do not embed task identifiers, phase numbers, or step numbers in code names, file names, comments, or commit messages. +9. Version numbers belong to the runtime revision field and migration manifests, not to type names. If a migration window forces two schemas to coexist, name both by role and remove the older schema in the next storage-layout revision. +10. Lifecycle and authority labels (`legacy`, `old`, `deprecated`, `retired`, `canonical`, `official`, `v1`, `v2`) never name code, files, or documentation. Keep exactly one optimal implementation. A proxy-upgrade storage mirror is named by what it holds (for example `ImportedTokenFlowConfig`) and is removed in the next storage-layout revision. +11. Prefer one common verb plus one concrete object for function names. Use direct verbs such as `get`, `set`, `check`, `read`, `write`, `load`, `save`, `add`, `remove`, `create`, `update`, `apply`, `build`, and `derive`; avoid multi-verb names, process jargon, and subjective lifecycle labels when one plain action states the behavior. `persist` is banned: write durable data with `save`/`write`, read it back with `load`/`read`; do not introduce `persist*`, `store` as a verb synonym for those, or other interchangeable synonyms for the same operation. +12. Name quantities as `__`, with the unit last: `remaining_backup_bytes`, `max_proof_bytes`, `processed_user_count`. Avoid sentence fragments such as `backup_bytes_remaining` and implementation terms such as `allocation` when the value is a validated size or count. +13. Use common verbs consistently: `get` reads an existing value; `load` assembles a domain value from durable storage; `read` decodes a file or byte stream; `build` assembles a composite value without persistence; `derive` computes a deterministic hash or identity from known inputs; `create` makes a new stored or runtime object; `set` replaces a whole value; `update` changes part of a value; `apply` executes a state transition; `validate` checks untrusted or serialized input; `ensure` enforces an internal invariant and returns an error; `check` returns a state classification or health result. Do not use these as interchangeable synonyms. +14. Order composite domain names from scope to object to representation: `checkpoint_tree_root`, `validator_tree_proof`, `user_leaf_hash`, `gathering_checkpoint_id`. State qualifiers precede the domain name (`current_`, `next_`, `last_committed_`, `remaining_`, `expected_`, `actual_`); collections use plural nouns and identifiers end in `_id` or `_ids`. ## Module Boundaries and Imports @@ -698,17 +703,19 @@ A change is rejected until any applicable item is corrected: ## Documentation Standards -1. `docs/` is the official developer-facing documentation: architecture, protocol, CLI reference, and verified procedures. It is split by audience. Public developer documentation lives under the mdBook-published tree (`src/SUMMARY.md` registration required). Internal developer documentation (devnet operations, verifier/circuit update procedures, debugging playbooks, incident postmortems) lives under `docs/src/dev/` and MUST NOT be registered in `src/SUMMARY.md` — it is repository-only and never published. Keep each topic consolidated in one document; do not fragment operational knowledge into many scattered files, and do not mix internal debugging records into public docs. Current GUTA pipeline internals: `docs/src/dev/reward-tree-circuits.md` (circuit reward layout and ASCII tree), `docs/src/dev/gatherers.md`, `docs/src/dev/processors.md`. - -2. Specs, reviews, and research documents must support factual claims with current `:` references. -3. Reviews accept verified facts or explicit open questions, not inference presented as evidence. -4. Test plans cover unit, integration, negative, and regression checks where applicable. -5. Acceptance criteria are executable commands or observable scenarios. -6. Mark inferred research statements explicitly as `Inference:` and list unchecked areas. -7. Separate `In Scope` and `Out of Scope` in every specification. -8. Operational documents are command-verified before commit: binaries, subcommands, flags, environment variables, RPC method names, ports, and configuration keys must match the current clap, serde, and network-configuration definitions. A nonexistent binary, flag, or environment variable in a document is a defect, not a style issue. -9. When explaining code to users, default to concise pseudocode plus the core function and parameter names. Show real implementation excerpts or full data structures only when needed to resolve ambiguity, prove a claim, or enable a concrete action. -10. Keep ordinary technical replies centered on the decision, observable behavior, material risk, and verification. +1. `docs/` is the official developer-facing documentation: architecture, protocol, CLI reference, and verified procedures. It is split by audience. +2. Public developer documentation lives under the mdBook-published tree (`src/SUMMARY.md` registration required). Internal developer documentation (devnet operations, verifier/circuit update procedures, debugging playbooks, incident postmortems) lives under `docs/src/dev/` and MUST NOT be registered in `src/SUMMARY.md` — it is repository-only and never published. +3. Keep each topic consolidated in one document; do not fragment operational knowledge into many scattered files, and do not mix internal debugging records into public docs. +4. Shared naming vocabulary is `docs/src/dev/TERMINOLOGY.md`. Current GUTA pipeline internals: `docs/src/dev/reward-tree-circuits.md` (circuit reward layout and ASCII tree), `docs/src/dev/gatherers.md`, `docs/src/dev/processors.md`. +5. Specs, reviews, and research documents must support factual claims with current `:` references. +6. Reviews accept verified facts or explicit open questions, not inference presented as evidence. +7. Test plans cover unit, integration, negative, and regression checks where applicable. +8. Acceptance criteria are executable commands or observable scenarios. +9. Mark inferred research statements explicitly as `Inference:` and list unchecked areas. +10. Separate `In Scope` and `Out of Scope` in every specification. +11. Operational documents are command-verified before commit: binaries, subcommands, flags, environment variables, RPC method names, ports, and configuration keys must match the current clap, serde, and network-configuration definitions. A nonexistent binary, flag, or environment variable in a document is a defect, not a style issue. +12. When explaining code to users, default to concise pseudocode plus the core function and parameter names. Show real implementation excerpts or full data structures only when needed to resolve ambiguity, prove a claim, or enable a concrete action. +13. Keep ordinary technical replies centered on the decision, observable behavior, material risk, and verification. ## Agent Communication Style @@ -718,15 +725,16 @@ A change is rejected until any applicable item is corrected: 4. Clarify module boundaries with an ownership table: which module owns which datum, who reads, who writes. 5. Use tables for classifications (mechanism vs necessity, component vs verdict) instead of narrative paragraphs. - ## Naming Quality Checklist -Every new name (function, file, directory, type, concept) must pass three tests. Any "no" means rename before landing. +Every new name (function, file, directory, type, concept) must pass three tests. Any "no" means rename before landing. Read `docs/src/dev/TERMINOLOGY.md` first; if that file already names the concept, reuse that word. 1. **Does the name say WHAT it does or contains?** The name must name a concrete domain concept, not a generic category. `objects/` fails (anything could be an object); `proposals/` passes (the directory contains proposals). `put_complete` fails ("complete" is an adjective, not an object); `store_proposal` passes ("store" is the action, "proposal" is the object). 2. **Can someone who has never seen the code understand it from the name alone?** `lookup` fails (lookup what? by what key?); `lookup_by_checkpoint` passes (self-contained). `active view` fails ("view" is a presentation term, not a domain concept); `applied set` passes ("applied" = committed to DB, "set" = the collection). 3. **Is there a more specific word that would be equally short?** If the answer is yes, use the more specific word. Never keep a vague name because renaming feels disruptive — vague names compound into unreadable codebases. + The pattern for functions: **verb + concrete object**. The pattern for files/directories: **concrete noun**. If the object slot is filled by a generic word (`data`, `result`, `obj`, `items`, `view`, `complete`), replace it with the actual domain noun. The same ban covers generic catch-all nouns: `metadata`, `manifest`, `info`, `payload`, `context`, `detail(s)`, `entry`, `item(s)`, `blob`, `misc` — each must be replaced by what the value actually is (e.g. `PsyProvingJobMetadata` holds reward-tree layout and dependencies → name it by those fields, not "metadata"; a reset marker file is a `reset_marker`, not a "manifest"). `metadata`/`manifest` are allowed only where the word names a real external API or an established on-disk format owned outside this workspace; new code must name the concrete content. `isolated`/`isolation` are also banned as names: say what the check actually does — untrusted records replayed against the authenticated baseline before any durable write — e.g. `verify_state_updates_against_baseline`, `baseline_replay`; do not name new functions, types, fields, or error strings with `isolat*`. `submission` is banned as a new name: name the concrete act (`end_cap_upload`, `proof_delivery`, `vote_publication`) or reuse the established wire/type name it wraps; `Submit*`/`AlreadySubmitted` survive only as existing external API/wire names owned by current shipped interfaces. + ## Git Commit Rules 1. Commit each independent task or milestone separately. diff --git a/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs b/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs deleted file mode 100644 index be7fcd86..00000000 --- a/client_prover/psy_cli/psy_user_cli/examples/derive_shield.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Derive a receiver shield address for private transfers. -//! -//! Usage: derive_shield -//! Prints the shield address in the canonical form accepted by -//! `psy_user_cli private-transfer --receiver` (QHashOut serde string round-trip). - -use plonky2::field::types::Field; -use psy_crypto::shield_address::derive_shield_address; -use serde_json::to_string; - -fn parse_arg(value: &str, name: &str) -> anyhow::Result { - value.parse::().map_err(|_| anyhow::anyhow!("invalid {name}: {value}")) -} - -fn main() -> anyhow::Result<()> { - let mut args = std::env::args().skip(1); - let user_id: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing user_id"))?, "user_id")?; - let random0: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing random0"))?, "random0")?; - let random1: u64 = parse_arg(&args.next().ok_or_else(|| anyhow::anyhow!("missing random1"))?, "random1")?; - let shield = derive_shield_address(user_id, random0, random1); - // serde round-trip: from_str parses exactly this string back. - println!("{}", to_string(&shield)?.replace('"', "")); - let _ = plonky2::field::goldilocks_field::GoldilocksField::ZERO; - Ok(()) -} diff --git a/client_prover/psy_cli/psy_user_cli/src/main.rs b/client_prover/psy_cli/psy_user_cli/src/main.rs index 44e82c15..1bfdadb5 100644 --- a/client_prover/psy_cli/psy_user_cli/src/main.rs +++ b/client_prover/psy_cli/psy_user_cli/src/main.rs @@ -226,7 +226,11 @@ fn command_paths(cli: &Cli) -> Vec<&str> { Commands::ProveTxTrace(args) => { push_session_paths(&mut paths, &args.session); paths.extend([args.session.rpc_config.as_str(), args.input.as_str()]); paths.extend(args.output.as_deref()); } Commands::PrivateTransfer(args) => paths.extend([args.rpc_config.as_str(), args.output.as_str()]), Commands::PrivateClaim(args) => { paths.push(&args.rpc_config); paths.extend(args.note_proof.as_deref()); } - Commands::DeriveNoteOwner(args) => paths.push(&args.rpc_config), + Commands::DeriveShield(args) => { + if args.private_key.is_some() { + paths.push(&args.rpc_config); + } + } Commands::ClaimDeposit(args) => { push_wallet_paths(&mut paths, &args.wallet); paths.extend([args.rpc_config.as_str(), args.deposit_proof.as_str()]); } Commands::Withdraw(args) => { push_wallet_paths(&mut paths, &args.wallet); paths.push(&args.rpc_config); } Commands::Deposit(args) => { paths.push(&args.rpc_config); paths.extend(args.deposit_proof_output.as_deref()); } @@ -698,7 +702,7 @@ async fn main() -> anyhow::Result<()> { } Commands::PrivateTransfer(args) => crate::subcommand::private_transfer::run(args).await?, Commands::PrivateClaim(args) => crate::subcommand::private_claim::run(args).await?, - Commands::DeriveNoteOwner(args) => crate::subcommand::shield_address::run(args).await?, + Commands::DeriveShield(args) => crate::subcommand::shield_address::run(args).await?, Commands::ClaimDeposit(args) => claim_deposit::run(args).await?, Commands::Deposit(args) => deposit::run(args).await?, Commands::Withdraw(args) => withdraw::run(args).await?, diff --git a/client_prover/psy_cli/psy_user_cli/src/result.rs b/client_prover/psy_cli/psy_user_cli/src/result.rs index 4efdf33b..0e3ecd2d 100644 --- a/client_prover/psy_cli/psy_user_cli/src/result.rs +++ b/client_prover/psy_cli/psy_user_cli/src/result.rs @@ -198,11 +198,13 @@ pub struct StatusResult { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NoteOwnerResult { - pub public_key: QHashOut, +pub struct ShieldAddressResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub public_key: Option>, pub user_id: u64, - pub note_owner: QHashOut, - pub nostr_npub: String, + pub shield_address: QHashOut, + #[serde(skip_serializing_if = "Option::is_none")] + pub nostr_npub: Option, } /// Public headers only. The trace payload, call data, witnesses, proofs, note @@ -251,7 +253,7 @@ pub enum CommandResult { CheckpointId(CheckpointIdResult), ClaimAmount(ClaimAmountResult), TxStatus(StatusResult), - NoteOwner(NoteOwnerResult), + ShieldAddress(ShieldAddressResult), TxTrace(TxTraceResult), Proofs(ProofsResult), Generic(GenericResult), @@ -288,7 +290,7 @@ impl CommandResult { Self::CheckpointId(v) => write_json_atomically(path, v), Self::ClaimAmount(v) => write_json_atomically(path, v), Self::TxStatus(v) => write_json_atomically(path, v), - Self::NoteOwner(v) => write_json_atomically(path, v), + Self::ShieldAddress(v) => write_json_atomically(path, v), Self::TxTrace(v) => write_json_atomically(path, v), Self::Proofs(v) => write_json_atomically(path, v), Self::Generic(v) => write_json_atomically(path, v), @@ -417,6 +419,24 @@ mod tests { } } + #[test] + fn shield_address_result_omits_optional_identity_on_user_id_path() { + let value = serde_json::to_value(ShieldAddressResult { + public_key: None, + user_id: 7, + shield_address: QHashOut::::from_values(1, 2, 3, 4), + nostr_npub: None, + }) + .unwrap(); + let object = value.as_object().unwrap(); + assert!(object.get("shield_address").is_some()); + assert_eq!(object.get("user_id"), Some(&serde_json::json!(7))); + assert!(!object.contains_key("public_key")); + assert!(!object.contains_key("nostr_npub")); + assert!(!object.contains_key("note_owner")); + assert!(!object.contains_key("private_key")); + } + #[test] fn typed_tree_root_result_serializes_its_payload() { let dir = temp_dir("tree-root"); diff --git a/client_prover/psy_cli/psy_user_cli/src/subcommand/args.rs b/client_prover/psy_cli/psy_user_cli/src/subcommand/args.rs index 12cd5864..f12ef8a5 100644 --- a/client_prover/psy_cli/psy_user_cli/src/subcommand/args.rs +++ b/client_prover/psy_cli/psy_user_cli/src/subcommand/args.rs @@ -795,11 +795,13 @@ pub struct WithdrawArgs { } #[derive(Clone, Args, Serialize, Deserialize)] -pub struct DeriveNoteOwnerArgs { +pub struct DeriveShieldArgs { #[clap(env, long, default_value = "config.json")] pub rpc_config: String, - #[clap(long, short = 'p')] - pub private_key: String, + #[clap(long, short = 'p', conflicts_with = "user_id", required_unless_present = "user_id")] + pub private_key: Option, + #[clap(long, conflicts_with = "private_key", required_unless_present = "private_key")] + pub user_id: Option, #[clap(long, default_value_t = 0)] pub random0: u64, #[clap(long, default_value_t = 0)] diff --git a/client_prover/psy_cli/psy_user_cli/src/subcommand/mod.rs b/client_prover/psy_cli/psy_user_cli/src/subcommand/mod.rs index 98836f5c..174085b7 100644 --- a/client_prover/psy_cli/psy_user_cli/src/subcommand/mod.rs +++ b/client_prover/psy_cli/psy_user_cli/src/subcommand/mod.rs @@ -145,8 +145,8 @@ pub enum Commands { PrivateTransfer(crate::subcommand::args::PrivateTransferArgs), /// Claim a private note from generated proof payload. PrivateClaim(crate::subcommand::args::PrivateClaimArgs), - /// Derive note owner hash from receiver pubkey and binding. - DeriveNoteOwner(crate::subcommand::args::DeriveNoteOwnerArgs), + /// Derive a shield address from a private key or a known user id. + DeriveShield(crate::subcommand::args::DeriveShieldArgs), /// Claim a bridge deposit on L2. Requires the deposit proof from /// psy-services. ClaimDeposit(crate::subcommand::args::ClaimDepositArgs), diff --git a/client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs b/client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs index 34c66390..9c0077f1 100644 --- a/client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs +++ b/client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs @@ -1,19 +1,19 @@ use std::str::FromStr; use nostr_sdk::prelude::{Keys, ToBech32}; -use plonky2::field::types::Field; use psy_client_common::{data::qhashout::QHashOut, ups::circuits::LocalCircuitType}; use psy_client_data::config::store_config::F; use psy_crypto::{ - hash::traits::hasher::{FieldQHasher, PoseidonHasher}, + hash::traits::hasher::PoseidonHasher, + shield_address::derive_shield_address, signature::zk::wallet::SimplePsyPrivateKey, }; use psy_prover::session::WalletSession; use sha2::{Digest, Sha256}; use crate::{ - result::{CommandResult, NoteOwnerResult}, - subcommand::args::DeriveNoteOwnerArgs, + result::{CommandResult, ShieldAddressResult}, + subcommand::args::DeriveShieldArgs, }; const NOSTR_PREFIX: &[u8] = b"psy-privacy-v0-nostr"; @@ -62,11 +62,19 @@ fn derive_nostr_npub(private_key: &str, random0: u64, random1: u64) -> anyhow::R Ok(keys.public_key().to_bech32()?) } -pub async fn run(args: DeriveNoteOwnerArgs) -> anyhow::Result { - let psy_config = psy_config::PsyConfigGoldilocks::from_file(&args.rpc_config)?; +pub async fn run(args: DeriveShieldArgs) -> anyhow::Result { + match (&args.private_key, args.user_id) { + (Some(private_key), None) => run_from_private_key(args.rpc_config, private_key, args.random0, args.random1).await, + (None, Some(user_id)) => run_from_user_id(user_id, args.random0, args.random1), + _ => anyhow::bail!("exactly one of --private-key or --user-id is required"), + } +} + +async fn run_from_private_key(rpc_config: String, private_key: &str, random0: u64, random1: u64) -> anyhow::Result { + let psy_config = psy_config::PsyConfigGoldilocks::from_file(&rpc_config)?; let rpc_config = psy_config.get_current_network()?.clone(); - let receiver_sk = QHashOut::::from_str(&args.private_key).map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?; + let receiver_sk = QHashOut::::from_str(private_key).map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?; let wallet_session = WalletSession::new(&rpc_config).await?; let zk_sig_fingerprint = wallet_session .circuit_info @@ -81,25 +89,99 @@ pub async fn run(args: DeriveNoteOwnerArgs) -> anyhow::Result { .copied() .ok_or_else(|| anyhow::anyhow!("No user id found for receiver public key"))?; - let note_owner = PoseidonHasher::q_hash_many(&[ - F::from_canonical_u64(receiver_user_id), - F::from_canonical_u64(1337), - F::from_canonical_u64(args.random0), - F::from_canonical_u64(args.random1), - ]); - - println!("public_key: {}", receiver_public_key); - println!("user_id: {}", receiver_user_id); - println!("random0: {}", args.random0); - println!("random1: {}", args.random1); - println!("note_owner: {}", note_owner); - let nostr_npub = derive_nostr_npub(&args.private_key, args.random0, args.random1)?; - println!("nostr_npub: {}", nostr_npub); - println!("private_address: {}#{}", note_owner, nostr_npub); - Ok(CommandResult::NoteOwner(NoteOwnerResult { - public_key: receiver_public_key, + let shield_address = derive_shield_address(receiver_user_id, random0, random1); + let nostr_npub = derive_nostr_npub(private_key, random0, random1)?; + print_shield(receiver_user_id, shield_address, Some(&nostr_npub)); + Ok(CommandResult::ShieldAddress(ShieldAddressResult { + public_key: Some(receiver_public_key), user_id: receiver_user_id, - note_owner, - nostr_npub, + shield_address, + nostr_npub: Some(nostr_npub), + })) +} + +fn run_from_user_id(user_id: u64, random0: u64, random1: u64) -> anyhow::Result { + let shield_address = derive_shield_address(user_id, random0, random1); + print_shield(user_id, shield_address, None); + Ok(CommandResult::ShieldAddress(ShieldAddressResult { + public_key: None, + user_id, + shield_address, + nostr_npub: None, })) } + +fn print_shield(user_id: u64, shield_address: QHashOut, nostr_npub: Option<&str>) { + println!("user_id: {}", user_id); + println!("shield_address: {}", shield_address); + if let Some(nostr_npub) = nostr_npub { + println!("nostr_npub: {}", nostr_npub); + println!("private_address: {}#{}", shield_address, nostr_npub); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::subcommand::{args::DeriveShieldArgs, Cli}; + use clap::Parser; + + fn parse(argv: &[&str]) -> Result { + let mut args = vec!["psy_user_cli"]; + args.extend(argv); + match Cli::try_parse_from(args)?.command { + crate::subcommand::Commands::DeriveShield(args) => Ok(args), + _ => panic!("expected DeriveShield"), + } + } + + #[test] + fn parse_rejects_missing_identity() { + assert!(parse(&["derive-shield", "--random0", "1", "--random1", "2"]).is_err()); + } + + #[test] + fn parse_rejects_both_identities() { + assert!(parse(&[ + "derive-shield", + "--private-key", + "aa", + "--user-id", + "7", + "--random0", + "1", + "--random1", + "2", + ]) + .is_err()); + } + + #[test] + fn parse_accepts_user_id_without_rpc() { + let args = parse(&["derive-shield", "--user-id", "7", "--random0", "1", "--random1", "2"]).unwrap(); + assert_eq!(args.user_id, Some(7)); + assert!(args.private_key.is_none()); + } + + #[test] + fn parse_accepts_private_key() { + let args = parse(&["derive-shield", "--private-key", "aa", "--random0", "1", "--random1", "2"]).unwrap(); + assert_eq!(args.private_key.as_deref(), Some("aa")); + assert!(args.user_id.is_none()); + } + + #[test] + fn user_id_path_matches_crypto_helper() { + let result = run_from_user_id(7, 1, 2).unwrap(); + match result { + CommandResult::ShieldAddress(value) => { + assert_eq!(value.user_id, 7); + assert_eq!(value.shield_address, derive_shield_address(7, 1, 2)); + assert!(value.public_key.is_none()); + assert!(value.nostr_npub.is_none()); + } + _ => panic!("expected ShieldAddress"), + } + } + +} diff --git a/docs/src/dev/TERMINOLOGY.md b/docs/src/dev/TERMINOLOGY.md new file mode 100644 index 00000000..c4b63901 --- /dev/null +++ b/docs/src/dev/TERMINOLOGY.md @@ -0,0 +1,235 @@ +# TERMINOLOGY + +Internal developer vocabulary for the PsyProtocol cohort. English only. +Not part of the published mdBook (`SUMMARY.md`). This file is the naming +authority for functions, variables, fields, and types. `AGENTS.md` requires +reading it before choosing a name. Role-specific docs still own their +pipelines. + +One concept, one term. Reuse an existing word. Do not add a synonym. If a +needed term is absent, add it here in the same change that introduces the +symbol. + +Current sources: `AGENTS.md` naming verbs; `docs/src/protocol/` circuit and +architecture docs; `docs/src/dev/processors.md`; `docs/src/dev/gatherers.md`; +`docs/src/dev/deposit-withdrawal.md`; `docs/src/dev/private-transfer.md`; +`psy_data/src/node/realm_processor.rs`; `psy_node_common/src/realm/processor/`. + +Pipeline internals stay in their owners: reward-tree layouts, gatherers, +processors, circuit operations. + +## 1. Verbs + +Use one verb for one act. Do not invent a synonym. + +| Verb | Means | Examples | +|---|---|---| +| `get` | Read an existing value | `get_latest_checkpoint_id` | +| `load` | Assemble a domain value from durable storage | `load_proposal`, `load_realm_memory_trees_from_db` | +| `read` | Decode a file or byte stream | `read_body_chunk`, `read_staged` | +| `build` | Assemble a composite value without persistence | planner job trees, `build_proposal_with_body` | +| `create` | Make a new stored or runtime object | `create_staged` | +| `recreate` | Abort a runtime object and create a replacement | `recreate_guta_gatherer` | +| `abort` | Cancel a runtime task | `abort_guta_gatherer` | +| `set` | Replace a whole value | `set_latest_checkpoint_id` | +| `update` | Change part of a value | `update_from_core_state` | +| `save` | Write durable bytes | `save_proposal` | +| `apply` | Execute a state transition | `apply_history_proposal`, `apply_proposal_ffs`, `apply_fast_forward_with_tree` | +| `commit` | Write the durable checkpoint records and marker | `commit_state` | +| `verify` | Check an untrusted or reconstructed candidate | `verify_history_transition`, `verify_history_proposal` | +| `validate` | Check untrusted or serialized input | wire/body checks | +| `ensure` | Enforce an internal invariant; return an error | `ensure_db_matches_coordinator_head` | +| `check` | Classify state or health | `get_database_check_state` | +| `sync` | Copy authenticated coordinator metadata locally | `sync_with_coordinator`, `sync_to_coordinator_checkpoint_id` | +| `install` | Verify staged bytes, atomically rename them into their destination, then fsync the parent directory | `ProposalBackup::install` | +| `stage` | Write a candidate that is not yet retained | `create_staged`, `stage_transition_blocks` | +| `derive` | Compute a deterministic hash or identity from known inputs | `derive_shield_address`, `derive-shield` | + +`persist` is banned. Do not add `store_*` as a verb synonym for `save`/`write`. + +## 2. Include versus apply versus commit + +These three words are not interchangeable. + +```text +Proposer submits GUTA + Proposal + Certificate + | + v +Coordinator include realm root appears in a coordinator checkpoint + | wait log: "Waiting for Coordinator to include Realm Root" + v +Realm apply FFS execute the proposal body against local trees + | apply_history_proposal / apply_proposal_ffs + v +Realm commit durable mappings, FFS rows, then set_latest_checkpoint_id + commit_state; last_committed_* advances +``` + +| Word | Owner | Meaning | Do not use it for | +|---|---|---|---| +| Include | Coordinator | The checkpoint tree now carries this realm root. Noun: **inclusion**. | Local durable writes. Function names that apply FFS. | +| Apply | Realm (or gatherer FastForward) | Execute the FFS / tree transition. Pair of **unapplied**. | The checkpoint marker write. | +| Commit | Local processor | `commit_state` writes records and the marker. Adjective: **last_committed_**. | Coordinator inclusion. A proposal that is only certified. | + +`included` appears as `included_checkpoint` on `UnappliedTransition`. That is the coordinator +checkpoint that already carries the transition. It is not a good function +adjective: the ready-path function *applies* FFS, then *commits* locally. + +Rejected names for the ready-path function: + +| Name | Why not | +|---|---| +| `commit_included_proposal_ffs` | `commit_state` already owns commit. | +| `apply_included_proposal_ffs` | `included` names coordinator inclusion, not the local act. | +| `apply_committed_proposal_ffs` | `committed` is `last_committed_*`, which this function produces, not consumes. | + +Chosen name: **`apply_proposal_ffs`**. Verb `apply`, object `proposal_ffs`, same +family as `apply_history_proposal` and `first_unapplied_transition`. + +Catch-up sibling: `apply_history_transitions` walks `C+1..=tip`. The ready path +applies the next unapplied proposal FFS once per `sync_and_verify`. + +`CheckpointIdentity` contains `checkpoint_id` and `checkpoint_leaf_hash`; the latter is the +coordinator checkpoint hash leaf, matching `checkpoint_sync_info.checkpoint_leaf_hash`. +The parameter name is `included`. + +## 3. Realm epochs + +`RealmProcessorCoreState` (`psy_data/src/node/realm_processor.rs`) keeps three +epochs. Name the epoch; do not invent a fourth. + +| Epoch | Fields | Meaning | +|---|---|---| +| Gathering | `gathering_*` | Live EndCap intake. Gatherer-owned trees. May sit on N+1 while N proves. | +| Processing | `processing_*` | The batch being proved / submitted. Speculative until commit. | +| Committed | `last_committed_*` | Durable head. Marker is `set_latest_checkpoint_id`. | + +`unique_pending_id` isolates one gathering / processing / committed batch. + +## 4. Protocol objects + +| Term | Meaning | +|---|---| +| PARTH | Parallelizable Account-based Recursive Transaction History. Hierarchical Merkle forest, not a single global state machine. | +| Coordinator | Single writer of the canonical checkpoint tree. | +| Realm | Shard. Two subs (`sub_id` 1 and 2) replicate one realm. | +| Processor | Long-lived loop: gather, prove, wait for inclusion, commit. | +| Gatherer | Background NATS consumer + planner. Owns the live in-memory tree. | +| Planner | Builds proving jobs and FFS for one gatherer cycle. | +| Edge | HTTP admission surface in front of a processor. EndCaps enter a Realm edge; GUTA submit enters the Coordinator edge. | +| Worker | Proves claimed jobs and submits tagged proofs back through the edge. | +| Relayer | L1/L2 bridge daemon (`psy_relayer_cli`): deposit append, withdrawal claim, Groth16. | +| EndCap | Final proof of one user proving session (`UPSStandardEndCapCircuit`). | +| GUTA | Global user-tree aggregator proof (realm circuit 63 at submit). | +| Proposal | Signed realm transition: old/new realm root plus body hash. | +| Proposal body | Finalizer output, proof, FFS bytes. Retained under `proposal_backups/bodies/`. | +| Certificate | Aggregated BLS votes for a proposal. | +| Vote | One validator signature on a proposal. | +| FFS | Fast-forward synchronization: Merkle-node / leaf updates. The payload followers apply. | +| FastForward | Gatherer command that applies FFS to the live tree, then recreates the builder. | +| CST | Coordinator checkpoint state-transition root job (circuit 32). | +| Checkpoint | Coordinator height `C`. Contiguous. Empty checkpoints still exist. | +| Realm transition | `(old_root, new_root)`. Local name: **`transition`**. Equal roots are leaf rewrites, not bodies. Not a `pair`. | +| Last-modified | Coordinator checkpoint where this realm root last changed. | +| UPS | User proving session. Local recursive proof chain that ends in an EndCap. | +| CFC | Contract function circuit. Verifiable contract method compiled by DPN. | +| DPN | Dapen. Compiles `.psy` contract methods into CFCs. | +| PI | Circuit public input. Recursive GUTA and coordinator proofs expose `H(header, R)`. | +| Tag | Worker claim tag hashed into every reward-tree node. | +| R | Reward-tree node value produced by the circuit for this job. | + +Rejected protocol names: `ProcessUserOp`, `AggregateUserOps`, `RealmStateTransition`, +`WrappedSignatureProof` / type 64 as a live circuit 63 child. + +## 5. State trees + +PARTH is a forest. Do not invent a second name for a tree that already has one. + +| Term | Meaning | +|---|---| +| CHKP | Checkpoint tree. Root of one block's global snapshot. | +| GUSR | Global user tree. Leaves are user accounts (`ULEAF`). | +| ULEAF | User leaf: public-key hash, balance, nonce, last checkpoint, `UCON` root. | +| UCON | Per-user contract tree. Maps contract id to that user's `CSTATE` root. | +| CSTATE | Per-user per-contract storage tree. | +| GCON | Global contract tree. Leaves are contract definitions (`CLEAF`). | +| CLEAF | Contract leaf: deployer hash, `CFT` root, `CSTATE` height. | +| CFT | Contract function tree. Function id to CFC fingerprint. | +| URT | User registration tree. | +| GDT | Global deposit tree. | +| GWT | Global withdrawal tree. | +| IMT-indexed tree | Previous `next_append_index != 0` for that `(user_id, contract_id)` contract-state tree. | +| Positional tree | Previous `next_append_index == 0`. Changed leaves on this tree do not require IMT records. | + +## 6. Privacy and bridge + +| Term | Meaning | +|---|---| +| Shield address | `PoseidonHash(user_id, 1337, r0, r1)`. Receiver identity for private notes and deposit claims. CLI: `derive-shield`. JSON field: `shield_address`. Not `note_owner`. | +| Note commitment | `PoseidonHash(nullifier_secret \|\| note_secret)`. | +| Nullifier hash | `PoseidonHash(nullifier_secret)`. Spend / claim tracker. | +| Private note | Shielded transfer note proved by `PrivateNoteInclusionCircuit`. | +| Deposit | L1→L2 lock via `Router` / `ERC20Gateway` / `Bridge`, claimed on L2 with `claim_deposit`. | +| Withdrawal | L2 burn then L1 release via Groth16 `batchClaimWithdrawal` / `claimPendingWithdrawal`. | +| Groth16 | Bridge wrapper proving system with circuit-specific setup material. | +| Plonky2 | Recursive proving backend for UPS, GUTA, CST, and live E2E. | +| JTMB | Test-only proving backend. Not rollback or live E2E evidence. | + +`derive-note-owner` and `note_owner` are retired CLI/result names for shield address. + +## 7. Runtime and CLI + +| Term | Meaning | +|---|---| +| `psy_node_cli` | Coordinator and Realm node binary. | +| `psy_worker_cli` | Job prover. | +| `psy_user_cli` | Wallet, contract, tree, bridge, and private-note CLI. | +| `psy_relayer_cli` | Bridge relayer. | +| `psy_dev_cli` | Operator CLI, including rollback. | +| Prove proxy | `psy_user_cli prove-proxy`. Groth16 helper for withdrawal claims. | +| ScyllaDB | Primary committed state backend. | +| NATS | Ephemeral gatherer queues. | +| RP | One role-local rollback plan, serialized as JSON. | + +## 8. Proposal backup + +On-disk directory: `local_checkpoints/realm_{R}_{S}/proposal_backups/`. + +| Symbol | Meaning | +|---|---| +| `ProposalBackup` | Runtime object over retained proposal bodies. | +| `proposal_backup` | Field / parameter holding that object. | +| `save_proposal` | Trusted local write: stage then install. | +| `create_staged` | Write a candidate file with RAII cleanup. | +| `install` | Verify staged bytes, atomically rename them into their destination, then fsync the parent directory; the destination is the transition-pair body path, and the backup's in-memory indexes are updated after the rename. | +| `retained` | A proposal body occupies its transition-pair path rather than a staging path; it survives close/reopen until replacement or removal. | +| `RetainedBodies` | In-memory indexes over retained proposal bodies. | +| `load_retained_transitions` | Assemble sorted transition pairs from retained filenames; body verification is separate. | +| `install_staged_proposals` | Test helper installing each staged proposal in fetch outcomes; returns the installed count. | +| `installed_proposal_count` | Number of proposals installed by that helper. | +| `build_proposal_with_body` / `build_proposal_with_body_at_checkpoint` | Test helpers deriving a proposal and its encoded body without persistence. | +| Retired `proposal_store/` | Abandoned directory name. Ignore it. | + +## 9. Recovery words + +| Term | Meaning | +|---|---| +| Catch-up | Walk coordinator checkpoints from `last_committed + 1` and apply missing transitions. | +| Recovery | Startup path when local state and coordinator head disagree; may rebuild backups. | +| Rollback | Operator-driven rewind of local durable head. Separate from catch-up. | +| Baseline replay | Re-verify FFS against the authenticated previous checkpoint before a vote or durable write. | +| Proof base | Checkpoint that authenticates a gatherer cycle start. | +| `load_changed_leaves_on_imt_indexed_trees` | Nonempty FFS-changed contract-state leaves on IMT-indexed trees. | + +## 10. Naming checklist + +1. The name says the object (`proposal`, `guta_gatherer`, `checkpoint`, `shield_address`), not a + category (`data`, `store`, `production`). +2. State qualifiers precede the object: `last_committed_realm_end_root`, + `processing_checkpoint_id`, `gathering_realm_start_root`. +3. Lifecycle labels (`legacy`, `old`, `deprecated`, `official`, `v1`) never + name code. +4. Prefer the existing verb table over a new synonym. +5. A verb-table gloss uses only registered verbs plus concrete OS mechanics, never a synonym. +6. A `RealmTransition` is `transition`. Do not name it `pair`. +7. Receiver identity for private notes and deposit claims is `shield_address`. Do not name it `note_owner`. diff --git a/docs/src/dev/bridge-common-operations.md b/docs/src/dev/bridge-common-operations.md index 67daf8cb..96d9704a 100644 --- a/docs/src/dev/bridge-common-operations.md +++ b/docs/src/dev/bridge-common-operations.md @@ -178,16 +178,16 @@ Derive the bridge-specific shield address using the same user and this run's fre ```bash ./target/release/psy_user_cli \ - --result-file "$RESULT_DIR/note-owner.json" \ - derive-note-owner \ + --result-file "$RESULT_DIR/shield-address.json" \ + derive-shield \ --private-key "$USER_PRIVATE_KEY" \ --rpc-config "$RPC_CONFIG" \ --random0 "$R0" \ --random1 "$R1" -export SHIELD_ADDRESS="$(jq -r '.note_owner' "$RESULT_DIR/note-owner.json")" +export SHIELD_ADDRESS="$(jq -r '.shield_address' "$RESULT_DIR/shield-address.json")" ``` -`derive-note-owner` resolves the same registered user and computes the bytes32 shield address from `R0` and `R1`; its structured `note_owner` is safe to use as the services query key (`client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs:65-104`). +`derive-shield` takes exactly one of `--private-key` or `--user-id`. The private-key path resolves the registered user over RPC and also prints the Nostr npub. The user-id path is offline. Structured `shield_address` is the services query key (`client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs:66-112`). The global `--result-file` is atomically published only on success and contains secret-free command results (`client_prover/psy_cli/psy_user_cli/src/subcommand/mod.rs:47-54`, `client_prover/psy_cli/psy_user_cli/src/result.rs:299-372`). diff --git a/docs/src/dev/gatherers.md b/docs/src/dev/gatherers.md index b6842a44..f4b290fd 100644 --- a/docs/src/dev/gatherers.md +++ b/docs/src/dev/gatherers.md @@ -6,6 +6,8 @@ ## Terminology +Shared verbs are in [TERMINOLOGY](TERMINOLOGY.md). Local terms: + | Term | Meaning | |---|---| | Gatherer | Background task that consumes one NATS ephemeral queue, feeds a planner, and returns a finalized snapshot for `process_block`. | @@ -147,10 +149,13 @@ A parked processor (`ProcessorState::Error`) stops every bound gatherer. That is ## Related Documents -- [Processors](processors.md) — who triggers finalize, publishes jobs, and commits. -- [Reward Tree Circuit Layouts](reward-tree-circuits.md) — modes the planners write. -- [RealmFinalizeGUTA BLS Authentication](realm-finalize-bls-auth.md) — official identity after gatherer finalize. -- [Realm P2P Validators](realm-p2p-validators.md) — who is scheduled to finalize. +| Document | Owns | +|---|---| +| [TERMINOLOGY](TERMINOLOGY.md) | Shared verbs; FFS, FastForward, gathering vs processing vs committed | +| [Processors](processors.md) | Who triggers finalize, publishes jobs, and commits | +| [Reward Tree Circuit Layouts](reward-tree-circuits.md) | Modes the planners write | +| [RealmFinalizeGUTA BLS Authentication](realm-finalize-bls-auth.md) | Official identity after gatherer finalize | +| [Realm P2P Validators](realm-p2p-validators.md) | Who is scheduled to finalize | ## File index diff --git a/docs/src/dev/private-transfer.md b/docs/src/dev/private-transfer.md index 0c674d20..83f2e401 100644 --- a/docs/src/dev/private-transfer.md +++ b/docs/src/dev/private-transfer.md @@ -56,19 +56,19 @@ The key cryptographic primitive is the `PrivateNoteInclusionCircuit`, which prov 3. The receiver has sufficient L2 PSY balance to pay the claim transaction fee (the claim itself burns a fee). 4. Release binaries are used for all operations. -### Step 1: Derive Receiver Note Owner +### Step 1: Derive Receiver Shield Address -The receiver derives a shielded note owner using their private key and two random values (`r0`, `r1`): +The receiver derives a shield address using two random values (`r0`, `r1`) and either their private key or a known `user_id`: ```text -note_owner = PoseidonHash(user_id, 1337, r0, r1) +shield_address = PoseidonHash(user_id, 1337, r0, r1) ``` -The receiver must remember `r0` and `r1` — they are required to claim the note. The `derive-note-owner` command outputs the note owner hash; obtain the receiver's Nostr public key separately (`client_prover/psy_cli/psy_user_cli/src/subcommand/args.rs:797-807`). +The receiver must remember `r0` and `r1` — they are required to claim the note. `psy_user_cli derive-shield` prints `shield_address`; the private-key path also prints the Nostr npub (`client_prover/psy_cli/psy_user_cli/src/subcommand/shield_address.rs:66-112`, `args.rs:798-808`). ### Step 2: Execute Private Transfer -The sender calls `private-transfer` with the receiver's note owner: +The sender calls `private-transfer` with the receiver's shield address: ```bash ./target/release/psy_user_cli private-transfer \ @@ -76,7 +76,7 @@ The sender calls `private-transfer` with the receiver's note owner: -p \ --contract-id \ --amount \ - --receiver \ + --receiver \ --note-root-slot 2147483649 \ --nostr-recipient-pubkey '' \ --output @@ -94,7 +94,7 @@ The sender calls `private-transfer` with the receiver's note owner: | Field | Type | Description | |-------|------|-------------| | `nullifier` | u64×4 | Nullifier hash of the spent note | -| `owner` | u64×4 | Receiver's note owner hash | +| `owner` | u64×4 | Receiver's shield address | | `amount` | string | Transfer amount | | `user_tree_root` | u64×4 | User tree root at proof time | | `checkpoint_id` | string | L2 checkpoint when proof was generated | @@ -191,9 +191,9 @@ Each private note has a unique `nullifier_hash = PoseidonHash(nullifier_secret)` assertion failed: nullifier already claimed ``` -### 3. Note Owner Binding +### 3. Shield Address Binding -The `owner` field in the `NoteProofOutput` is the receiver's note owner hash (`PoseidonHash(user_id, 1337, r0, r1)`). The claiming user must match this owner. Using a different private key or different `r0`/`r1` values results in: +The `owner` field in the `NoteProofOutput` is the receiver's shield address (`PoseidonHash(user_id, 1337, r0, r1)`). The claiming user must match this address. Using a different private key or different `r0`/`r1` values results in: ```text receiver does not match claiming user @@ -211,7 +211,7 @@ In the proving session, the external proof (note inclusion) must be inserted **b | Error | Cause | Resolution | |-------|-------|------------| -| `receiver does not match claiming user` | Wrong private key or wrong `r0`/`r1` | Ensure the receiver key and randoms match the note owner | +| `receiver does not match claiming user` | Wrong private key or wrong `r0`/`r1` | Ensure the receiver key and randoms match the shield address | | `nullifier already claimed` | Note was already claimed by someone | Check claim status before attempting | | `insufficient balance for fee` | Receiver has no L2 PSY for gas | Fund the receiver through `docs/src/dev/common-operations.md` Section 5.1; genesis `simple_mint` is not available to devnet wallets | | `note proof deserialization failed` | Corrupted or wrong format proof file | Regenerate the proof file | diff --git a/docs/src/dev/processors.md b/docs/src/dev/processors.md index 78120c3a..8550e3c4 100644 --- a/docs/src/dev/processors.md +++ b/docs/src/dev/processors.md @@ -6,6 +6,8 @@ ## Terminology +Shared verbs are in [TERMINOLOGY](TERMINOLOGY.md). Local terms: + | Term | Meaning | |---|---| | Processor | Long-lived async loop that gathers one batch, publishes proving jobs, and commits durable state. | @@ -178,11 +180,14 @@ Root-proof waits must observe the persisted proof store, not only a queue-comple ## Related Documents -- [Gatherers](gatherers.md) — who owns the trees and the N/N+1 seam. -- [Reward Tree Circuit Layouts](reward-tree-circuits.md) — part-1 and CST reward nodes. -- [RealmFinalizeGUTA BLS Authentication](realm-finalize-bls-auth.md) — submit/admit gate. -- [Realm P2P Validators](realm-p2p-validators.md) — scheduled proposer. -- [Devnet Lifecycle](devnet_lifecycle.md) — stack start/stop; processors are not restarted individually. +| Document | Owns | +|---|---| +| [TERMINOLOGY](TERMINOLOGY.md) | Shared verbs; include vs apply vs commit; ready-path `apply_proposal_ffs` | +| [Gatherers](gatherers.md) | Who owns the trees and the N/N+1 seam | +| [Reward Tree Circuit Layouts](reward-tree-circuits.md) | Part-1 and CST reward nodes | +| [RealmFinalizeGUTA BLS Authentication](realm-finalize-bls-auth.md) | Submit/admit gate | +| [Realm P2P Validators](realm-p2p-validators.md) | Scheduled proposer | +| [Devnet Lifecycle](devnet_lifecycle.md) | Stack start/stop; processors are not restarted individually | ## File index diff --git a/docs/src/dev/realm-recovery-e2e.md b/docs/src/dev/realm-recovery-e2e.md index 24646a2c..c98b8876 100644 --- a/docs/src/dev/realm-recovery-e2e.md +++ b/docs/src/dev/realm-recovery-e2e.md @@ -107,7 +107,7 @@ Failure signatures and what they mean: | `dropped unauthenticated Realm vote` | a vote arrived for a proposal whose vote-auth context is not registered locally | `drive.rs:328`; expected on the follower for the proposer's self-vote, abnormal if the follower never votes | | `follower baseline replay rejected proposal=` | the follower's baseline replay rejected the proposal (coverage or root binding) | `realm_p2p.rs:633`; reason comes from `recovery.rs` | | `timed out: wait_votes` | proposer never reached the replication threshold; 120 s deadline | `runner.rs:122` | -| `InvalidStateUpdates: ... has no IMT record` | nonzero changed leaf on a tree classified as IMT-managed lacks its required IMT record; see limitations below, not a universal positional-write rejection | `psy_node_common/src/realm/processor/recovery.rs:389-393,570-603` | +| `InvalidStateUpdates: ... has no IMT record` | nonzero changed leaf on an IMT-indexed tree lacks its required IMT record; see limitations below, not a universal positional-write rejection | `psy_node_common/src/realm/processor/recovery.rs:389-393,570-603` | ## H — lagging follower body catch-up @@ -481,7 +481,7 @@ proposers without a harness change. a no-op for production realms. - Per-tree pairing classifies a tree using a **positive LIVE next-append pointer**, then requires IMT records for its nonzero changed leaves, - including new indices on an already-managed tree + including new indices on an IMT-indexed tree (`psy_node_common/src/realm/processor/recovery.rs:570-603,389-393`). Zero-valued changed leaves remain exempt. The pointer read is not checkpoint-versioned @@ -604,3 +604,83 @@ realm 1; its user leaf on `:13390/:13391` stayed `0e49b588…` at both C2035 and C2036 (`inclusion-rpc.json`; token contract leaf already `…0001` from the earlier F transfers). +## Rename-head rerun (2026-09-17) + +Fresh Plonky2 stack after HEAD `e5d03fd5` (`recovery/` → `ffs/`). Anvil +state and localhost deployments were absent, so this is a new chain, not +`PURGE=0` resume. Release CLIs rebuilt; `make run-all` in `tmux` pane +`%12`. Evidence under `e2e-evidence/round3-*` (wallet files not committed). +Case 3 not run. + +`[COORD_CREATE] processor new done` 22:56:47Z; +`[REALM_CREATE] processor new done` r0s1 22:58:41, r1s1 22:59:31, +r0s2 23:00:21, r1s2 23:01:11. Prove-proxy +`[CFLI:PSY_PROVE_PROXY_STARTED][0.0.0.0:9999]` before F. + +### F — fee-reserved sequence — PASS + +Fresh zk wallet registered as user **1966080** +(`public_key_hash=9882911b…`, `get-user-id.json`; wallet-create +`9256a8fd…` is unused). Genesis user 0 `faucet(1966080, 3000006100)`: +first EndCap `tx=f1f377fc…` timed out (`grant.stderr.txt`); user-0 leaf +changed at C**61** to `e3fae7f6…` (nonce 0→1, balance 0→3000) instead +of the submitted hash (`first-grant.json`). Not replayed. Second call +from that leaf confirmed C**87** `tx=1ec4daa8…` (`grant.json`). +Recipient `simple_claim([0])` C**92** `tx=80f1b918…`. +`simple_transfer([1,50])` C**97** `tx=5a42d8c4…`. +`simple_transfer([1,49])` C**102** `tx=851d5057…`. Fourth +`simple_transfer([1,10])` **confirmed** C**106** `tx=99f46732…` because +leftover token still covered fee+10 (`xfer10.json`). Fifth +`simple_transfer([1,2000004092])` (leftover+1) failed at trace: +`assertion failed: insufficient balance (left: 0, right: 1)` +(`xfer-over.stderr.txt`). Five-edge roots equal at 87, 92, 97, 102, +106 (`*-rpc.json`). Pipeline grep: `pipeline-grep.txt`. + +### H — lagging follower body catch-up — PASS + +SIGSTOP r0s2 PID **3294194** at 23:17:41Z tip **120** (`freeze.json`, +`/proc` `T`). While frozen, `:13381` stayed 120. Epoch **14** (anchor +139) computed r0 `sub_1`. User-0 EndCap submitted 23:21:10Z +`tx=67a15b29…` then CLI timed out (`xfer.stderr.txt`); that hash +`2849e0f7…` never appeared on `:13380`. r0s1 published proposal +`95d8835f…` epoch 14 target 143 and committed C**144**. Operator/user-0 +leaf on `:13380` stayed `4cb2adb8…` through 143 and became `a540849a…` +at 144 (`inclusion-rpc.json`). SIGCONT 23:24:55Z (`cont.json`). r0s2 +logged `Realm P2P proposal body complete` proposal=`95d8835f…` then +`Committed coordinator processor state for checkpoint ID: 144` +(`catchup-log.txt`). Body +file `9f3156d0…_74269643…` (142020 bytes) appeared on r0s1 (mtime +23:21:14Z) and r0s2 (mtime 23:24:55Z); `cmp` identical +(`body-cmp.json`). Gossip `body_len=141806`; docs cite disk `cmp`. +Five-edge roots at 144 identical (`catchup-rpc.json`); +`no peer offered pair` count 0 (`ghost.txt`). + + +### Case 2 — missed epoch nonempty takeover — FAIL + +SIGSTOP r0s2 PID **3294194** at 23:31:43Z tip **207** epoch **20** +(computed r0 `sub_2`). Coordinator tip advanced 207→**219+** while that +processor stayed `T`; `:13381` stayed 207 (`freeze.json`, +`epoch-scan.json`). Epoch **21** (anchor 209) computed r0 `sub_2` +(`epoch-scan.json`). r0s1 logged only +`No GUTA jobs to process in this block, skipping.` (`process_block.rs:446`, +`r0s1-skip.txt`); +last `scheduled proposer realm=0 sub_id=1` remains 23:21:14Z epoch 14 +(`r0s1-scheduled.txt`). +Last r0s2 `scheduled proposer` remains 23:11:26Z epoch 8 +(`r0s2-scheduled.txt`). SIGCONT 23:34:03Z; five edges 220 then 221. +**Tip continued is not rotation takeover.** Nonempty sub_1 proposal +during the pause was not obtained. + +### W — realm proving-worker freeze — PASS + +| step | evidence | +|---|---| +| W0 | Five edges **223**. Realm worker PID **3296594** `S`; `logs/worker_0_logs.txt` size 26515. Coordinator workers 3290766/3291435 `S`. `w0.json`. | +| W1 | SIGSTOP 3296594 at 23:34:41Z; `/proc` `T`. Size stayed 26515 for 8s. Coordinator workers `S`; tip 223→224 (`w1.json`). | +| W2 | User-0 `simple_transfer` CLI timed out. r0s2 `Realm worker publication acknowledged` 23:35:36Z checkpoint 230 unique_pending_id **509** (`w2-ack.txt`). Worker log did not grow before SIGCONT. | +| W3 | Five `psy_get_checkpoint_global_state_roots[223]` identical (`user_tree_root=c44af9cd…`). Live tips during wait: `:13381`=230 others 237+. No 23:34–23:37 Fatal/RESTART (`w3-rpc.json`, `w3-fatals.txt`). | +| W4 | Frozen-window identity r0 `sub_2` epoch 23 (anchor 229). Epoch **24** (anchor 239) computed r0 `sub_1` (`w4-scan.json`). Not wall-clock. | +| W5 | SIGCONT 23:37:06Z; `T`→`S`; `size_before=26515` (`w5-cont.json`). `proving start` goal **509** GUTASingleEndCap (`w5-proving.txt`). | +| W6 | Same-epoch proposer r0 `sub_2` epoch 23. 23:37:09Z `scheduled proposer realm=0 sub_id=2 epoch=23 target=230`; certificate `signers=[2]`; follower r0s1 `proposal start accepted` + `proposal body complete` proposal=`cde5de6d…` body_len=**142689**; r0s2 `Committed new realm block with checkpoint_id = 241` (`w6-commit.txt`, `w6-body.txt`). Operator/user-0 leaf on `:13380/:13381` first equals `0069e3ee…` at C241 (C240 still `a540849a…`). Five-edge roots at 241 identical (`w6-rpc.json`). | + diff --git a/docs/src/dev/rollback-operations.md b/docs/src/dev/rollback-operations.md index d67d7d9d..58796d97 100644 --- a/docs/src/dev/rollback-operations.md +++ b/docs/src/dev/rollback-operations.md @@ -41,7 +41,7 @@ sequenceDiagram participant CLI as Release psy_dev_cli participant Stores as Retained stores Operator->>Supervisor: 1. make rollback-stop - Supervisor-->>Operator: 2. Applications stopped; sentinel written + Supervisor-->>Operator: 2. Applications stopped, sentinel written loop Coordinator and four Realm identities Operator->>CLI: 3. --generate with unique RP path CLI->>Stores: 4. Read target, mappings, backups, high-water @@ -55,7 +55,7 @@ sequenceDiagram end Operator->>Operator: 10. Require all phases completed and external recovery complete Operator->>Supervisor: 11. make rollback-resume - Supervisor-->>Operator: 12. Saved applications ready; sentinel removed + Supervisor-->>Operator: 12. Saved applications ready, sentinel removed ``` Generation reads authoritative stores and backups; execution writes the checkpoint marker last and persists phase progress (`psy_cli/psy_dev_cli/src/subcommand/rollback/generate.rs:301-340`; `psy_node_common/src/rollback/executor.rs:130-203`). diff --git a/docs/src/rpc/UserCli.md b/docs/src/rpc/UserCli.md index 8486adc6..15184d39 100644 --- a/docs/src/rpc/UserCli.md +++ b/docs/src/rpc/UserCli.md @@ -74,7 +74,7 @@ Client CLI for wallet, contract deploy/call, tree/metadata queries, proving help | `prove-tx-trace` | | `private-transfer` | | `private-claim` | -| `derive-note-owner` | +| `derive-shield` | | `claim-deposit` | | `withdraw` | | `deposit` | @@ -92,7 +92,7 @@ Client CLI for wallet, contract deploy/call, tree/metadata queries, proving help | Proving | `local-prover`, `prove-proxy`, `generate-tx-trace`, `prove-tx-trace`, `get-user-end-cap-common-data` | | Rewards / jobs | `get-claim-amount`, `batch-claim`, `claim-rewards`, `get-checkpoint-id-for-unique-pending-id`, `generate-batch-proof-miner-reward-proofs` | | Bridge | `deposit`, `claim-deposit`, `withdraw`, `claim-withdrawal` | -| Privacy | `private-transfer`, `private-claim`, `derive-note-owner` | +| Privacy | `private-transfer`, `private-claim`, `derive-shield` | | Misc | `tx`, `faucet-server`, `get-psy-sdc-fingerprint`, `get-user-event-data`, `get-user-leaf` | ## Source diff --git a/psy_cli/psy_node_cli/src/node/realm_p2p.rs b/psy_cli/psy_node_cli/src/node/realm_p2p.rs index 1e832643..c9e3af7a 100644 --- a/psy_cli/psy_node_cli/src/node/realm_p2p.rs +++ b/psy_cli/psy_node_cli/src/node/realm_p2p.rs @@ -29,8 +29,8 @@ use psy_node_common::{ }, processor::{ consensus::{decode_proposal_state_updates, sign_vote, verify_proposal_submission}, - proposal_store::ProposalStore, - recovery::BaselineReplayRequest, + proposal_backup::ProposalBackup, + ffs::BaselineReplayRequest, }, }, }; @@ -530,7 +530,7 @@ pub fn spawn_processor_realm_network( local_sub_id: u16, validator_store: Arc, proof_verifier: Arc, - proposal_store: Arc, + proposal_backup: Arc, validator_leaves: Vec, commands: RealmNetworkCommands, bls_secret: psy_data::p2p::BlsSecretKey, @@ -564,7 +564,7 @@ pub fn spawn_processor_realm_network( let proposal_id = proposal.proposal_id; let validator_store = validator_store.clone(); let proof_verifier = proof_verifier.clone(); - let proposal_store = proposal_store.clone(); + let proposal_backup = proposal_backup.clone(); let vote_enabled = vote_enabled.clone(); let baseline_replay = baseline_replay.clone(); let commands = commands.clone(); @@ -613,7 +613,7 @@ pub fn spawn_processor_realm_network( proposer_user_id, proof_verifier.as_ref(), )?; - proposal_store.save_proposal(&proposal, body.as_bytes()).await?; + proposal_backup.save_proposal(&proposal, body.as_bytes()).await?; if !vote_enabled.load(Ordering::Acquire) { return Ok(()); } @@ -655,7 +655,7 @@ pub fn spawn_processor_realm_network( }); } RealmNetworkEvent::LookupReceived { request, reply, .. } => { - let response = match proposal_store.lookup_proposal(&request).await { + let response = match proposal_backup.lookup_proposal(&request).await { Ok(response) => response, Err(error) => { tracing::warn!("ProposalLookup serve failed error={error:#}"); @@ -665,7 +665,7 @@ pub fn spawn_processor_realm_network( let _ = reply.send(response); } RealmNetworkEvent::DirectBodyReceived { request, reply, .. } => { - match proposal_store.read_body_chunk(&request).await { + match proposal_backup.read_body_chunk(&request).await { Ok(response) => { let _ = reply.send(response); } diff --git a/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs b/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs index 5d0f6d5f..5db7664d 100644 --- a/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs +++ b/psy_cli/psy_node_cli/src/node/startup_plonky2_scylla.rs @@ -364,8 +364,8 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce let coordinator_client = PsyRealmCoordinatorClientAPI::::new( http_client, ); - let proposal_store = Arc::new( - psy_node_common::realm::processor::proposal_store::ProposalStore::open( + let proposal_backup = Arc::new( + psy_node_common::realm::processor::proposal_backup::ProposalBackup::open( config.get_proposal_backups_path(), ) .await?, @@ -397,7 +397,7 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce realm_sub_id, validator_store, proof_verifier.clone(), - proposal_store.clone(), + proposal_backup.clone(), validator_leaves, commands.clone(), load_bls_secret_key(bls_key_path)?, @@ -420,7 +420,7 @@ pub async fn run_startup_plonky2_scylla_realm_processor_node(config: &RealmProce circuit_fingerprint_config, Arc::new(coordinator_client), proof_verifier, - proposal_store, + proposal_backup, Some(commands.clone()), ) .await?; diff --git a/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs b/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs index 749ce3fb..f2b5006a 100644 --- a/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs +++ b/psy_cli/psy_node_cli/src/node/startup_processor_jtmb_scylla.rs @@ -163,8 +163,8 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(co http_client, ); tracing::info!("[REALM_BOOT] creating realm processor"); - let proposal_store = Arc::new( - psy_node_common::realm::processor::proposal_store::ProposalStore::open( + let proposal_backup = Arc::new( + psy_node_common::realm::processor::proposal_backup::ProposalBackup::open( config.get_proposal_backups_path(), ) .await?, @@ -196,7 +196,7 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(co realm_sub_id, validator_store, proof_verifier.clone(), - proposal_store.clone(), + proposal_backup.clone(), validator_leaves, commands.clone(), load_bls_secret_key(bls_key_path)?, @@ -219,7 +219,7 @@ pub async fn run_startup_jtmb_poseidon_goldilocks_scylla_realm_processor_node(co circuit_fingerprint_config, Arc::new(coordinator_client), proof_verifier, - proposal_store, + proposal_backup, Some(commands.clone()), ) .await?; diff --git a/psy_data/src/p2p/messages.rs b/psy_data/src/p2p/messages.rs index c8c79070..d0282e7e 100644 --- a/psy_data/src/p2p/messages.rs +++ b/psy_data/src/p2p/messages.rs @@ -695,11 +695,11 @@ impl ProtocolEncode for BodyChunkResponse { } } -pub const PROPOSAL_LOOKUP_MAX_PAIRS: usize = 256; -pub const PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR: usize = 2; -/// chain_id(8) + realm_id(4) + pair_count(4) + pair_count * (old_root(32) + new_root(32)). -pub const PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES: usize = 16 + PROPOSAL_LOOKUP_MAX_PAIRS * 64; -pub const PROPOSAL_LOOKUP_WINDOW_PAIRS: usize = +pub const PROPOSAL_LOOKUP_MAX_TRANSITIONS: usize = 256; +pub const PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION: usize = 2; +/// chain_id(8) + realm_id(4) + transition_count(4) + transition_count * (old_root(32) + new_root(32)). +pub const PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES: usize = 16 + PROPOSAL_LOOKUP_MAX_TRANSITIONS * 64; +pub const PROPOSAL_LOOKUP_WINDOW_TRANSITIONS: usize = (MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES - 5) / (64 + 1 + PROPOSAL_WIRE_BYTES); #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -719,7 +719,7 @@ pub enum ProposalLookupStatus { pub struct ProposalLookupRequest { pub chain_id: u64, pub realm_id: u32, - pub pairs: Vec, + pub transitions: Vec, } impl ProposalLookupRequest { @@ -727,20 +727,20 @@ impl ProposalLookupRequest { let chain_id = reader.read_u64()?; let realm_id = reader.read_u32()?; let count = reader.read_u32()? as usize; - if count == 0 || count > PROPOSAL_LOOKUP_MAX_PAIRS { - return Err(ProtocolError::Message("invalid ProposalLookup pair count")); + if count == 0 || count > PROPOSAL_LOOKUP_MAX_TRANSITIONS { + return Err(ProtocolError::Message("invalid ProposalLookup transition count")); } if reader.remaining() < count * 64 { - return Err(ProtocolError::unexpected_eof("ProposalLookup pairs")); + return Err(ProtocolError::unexpected_eof("ProposalLookup transitions")); } - let mut pairs = Vec::with_capacity(count); + let mut transitions = Vec::with_capacity(count); for _ in 0..count { - pairs.push(RealmTransition { + transitions.push(RealmTransition { old_root: reader.read_bytes_32()?, new_root: reader.read_bytes_32()?, }); } - Ok(Self { chain_id, realm_id, pairs }) + Ok(Self { chain_id, realm_id, transitions }) } pub fn decode_exact(bytes: &[u8]) -> ProtocolResult { @@ -752,10 +752,10 @@ impl ProtocolEncode for ProposalLookupRequest { fn protocol_encode(&self, out: &mut Vec) { write_u64(out, self.chain_id); write_u32(out, self.realm_id); - write_u32(out, self.pairs.len() as u32); - for pair in &self.pairs { - write_fixed(out, &pair.old_root); - write_fixed(out, &pair.new_root); + write_u32(out, self.transitions.len() as u32); + for transition in &self.transitions { + write_fixed(out, &transition.old_root); + write_fixed(out, &transition.new_root); } } } @@ -798,7 +798,7 @@ impl ProposalLookupResponse { tag => return Err(ProtocolError::UnknownTag { ty: "ProposalLookupStatus", tag }), }; let count = reader.read_u32()? as usize; - if count > PROPOSAL_LOOKUP_MAX_PAIRS { + if count > PROPOSAL_LOOKUP_MAX_TRANSITIONS { return Err(ProtocolError::Message("invalid ProposalLookup entry count")); } if reader.remaining() < count * 65 { @@ -813,7 +813,7 @@ impl ProposalLookupResponse { new_root: reader.read_bytes_32()?, }; let candidate_count = reader.read_u8()? as usize; - if candidate_count > PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR { + if candidate_count > PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION { return Err(ProtocolError::Message("invalid ProposalLookup candidate count")); } wire_bytes += 65 + candidate_count * PROPOSAL_WIRE_BYTES; @@ -851,7 +851,7 @@ impl ProtocolEncode for ProposalLookupResponse { fn protocol_encode(&self, out: &mut Vec) { let mut wire_bytes = 5; let mut count = 0; - for entry in self.entries.iter().take(PROPOSAL_LOOKUP_MAX_PAIRS) { + for entry in self.entries.iter().take(PROPOSAL_LOOKUP_MAX_TRANSITIONS) { let entry_bytes = 65 + entry.candidates.len() * PROPOSAL_WIRE_BYTES; if wire_bytes + entry_bytes > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { break; @@ -1510,14 +1510,14 @@ mod tests { mod proposal_lookup_tests { use super::*; - fn pair(index: u8) -> RealmTransition { + fn transition(index: u8) -> RealmTransition { RealmTransition { old_root: [index; 32], new_root: [index.wrapping_add(1); 32] } } fn entries(count: usize, candidates: usize) -> Vec { let proposal = proposal_from_parts(1, 2, 3, 0, [1; 32], [2; 32], [3; 32], [4; 32], [5; 32]); (0..count).map(|index| ProposalLookupEntry { - transition: pair(index as u8), + transition: transition(index as u8), candidates: vec![proposal.clone(); candidates], }).collect() } @@ -1526,7 +1526,7 @@ mod proposal_lookup_tests { fn request_window_bounds_and_canonical_length() { let mut request = ProposalLookupRequest { chain_id: 1, realm_id: 2, - pairs: (0..PROPOSAL_LOOKUP_MAX_PAIRS).map(|index| pair(index as u8)).collect(), + transitions: (0..PROPOSAL_LOOKUP_MAX_TRANSITIONS).map(|index| transition(index as u8)).collect(), }; let bytes = request.protocol_encode_to_vec(); assert_eq!(bytes.len(), 16 + 256 * 64); @@ -1541,15 +1541,15 @@ mod proposal_lookup_tests { invalid[12..16].copy_from_slice(&count.to_le_bytes()); assert!(ProposalLookupRequest::decode_exact(&invalid).is_err()); } - request.pairs.truncate(1); + request.transitions.truncate(1); assert_eq!(request.protocol_encode_to_vec().len(), 80); assert_eq!(ProposalLookupRequest::decode_exact(&request.protocol_encode_to_vec()).unwrap(), request); } #[test] fn response_budget_preserves_complete_prefix() { - assert_eq!(PROPOSAL_LOOKUP_WINDOW_PAIRS, 58); - let response = ProposalLookupResponse::candidates(entries(PROPOSAL_LOOKUP_WINDOW_PAIRS, 1)); + assert_eq!(PROPOSAL_LOOKUP_WINDOW_TRANSITIONS, 58); + let response = ProposalLookupResponse::candidates(entries(PROPOSAL_LOOKUP_WINDOW_TRANSITIONS, 1)); let bytes = response.protocol_encode_to_vec(); assert_eq!(bytes.len(), 5 + 58 * (65 + PROPOSAL_WIRE_BYTES)); assert!(bytes.len() <= MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES); diff --git a/psy_data/src/p2p/mod.rs b/psy_data/src/p2p/mod.rs index 5197d841..93fcfa95 100644 --- a/psy_data/src/p2p/mod.rs +++ b/psy_data/src/p2p/mod.rs @@ -35,8 +35,8 @@ pub use messages::{ proposal_from_parts, vote_message, Certificate, BodyChunkRequest, BodyChunkResponse, EndCapForwardHeader, EndCapForwardResponse, EndCapRejectReason, Proposal, ProposalLookupEntry, ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, ProposalPart, - RealmFinalizeOutputBytes, RealmFinalizeSubmitCode, Vote, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, - PROPOSAL_LOOKUP_MAX_PAIRS, PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES, PROPOSAL_LOOKUP_WINDOW_PAIRS, + RealmFinalizeOutputBytes, RealmFinalizeSubmitCode, Vote, PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION, + PROPOSAL_LOOKUP_MAX_TRANSITIONS, PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES, PROPOSAL_LOOKUP_WINDOW_TRANSITIONS, }; pub use node_id::NodeId; pub use validator_leaf::ValidatorLeaf; diff --git a/psy_node_common/src/realm/network/codec.rs b/psy_node_common/src/realm/network/codec.rs index 4c767722..b2ee2fa2 100644 --- a/psy_node_common/src/realm/network/codec.rs +++ b/psy_node_common/src/realm/network/codec.rs @@ -3,7 +3,7 @@ //! //! Three request/response protocols are wired: //! - `/psy/realm/proposal-body/1` — bounded proposal body range exchange. -//! - `/psy/realm/proposal-lookup/1` — windowed root-pair proposal lookup. +//! - `/psy/realm/proposal-lookup/1` — windowed realm-transition proposal lookup. //! - `/psy/realm/end-cap-forward/2` — EndCap forward stream (56-byte header //! followed by `end_cap_input_len` input bytes and `proof_len` proof bytes); //! version 2 carries the 18-byte typed rejection response. @@ -21,7 +21,7 @@ use psy_data::p2p::{ ProposalLookupRequest, ProposalLookupResponse, ProtocolEncode, BODY_CHUNK_REQUEST_WIRE_BYTES, BODY_CHUNK_MAX_BYTES, END_CAP_FORWARD_HEADER_WIRE_BYTES, END_CAP_FORWARD_RESPONSE_WIRE_BYTES, MAX_END_CAP_FORWARD_BYTES, - MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, PROPOSAL_LOOKUP_MAX_PAIRS, + MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES, PROPOSAL_LOOKUP_MAX_TRANSITIONS, PROPOSAL_LOOKUP_REQUEST_MAX_WIRE_BYTES, }; use std::{fmt, io}; @@ -140,10 +140,10 @@ impl Codec for ProposalLookupCodec { where T: AsyncWrite + Unpin + Send, { - if request.pairs.is_empty() || request.pairs.len() > PROPOSAL_LOOKUP_MAX_PAIRS { - return Err(invalid_data("invalid ProposalLookup pair count")); + if request.transitions.is_empty() || request.transitions.len() > PROPOSAL_LOOKUP_MAX_TRANSITIONS { + return Err(invalid_data("invalid ProposalLookup transition count")); } - let expected = 16 + request.pairs.len() * 64; + let expected = 16 + request.transitions.len() * 64; let bytes = request.protocol_encode_to_vec(); if bytes.len() != expected { return Err(invalid_data("invalid ProposalLookup request length")); @@ -162,14 +162,14 @@ impl Codec for ProposalLookupCodec { { let has_candidates = response.entries.iter().any(|entry| !entry.candidates.is_empty()); if response.entries.iter().any(|entry| { - entry.candidates.len() > psy_data::p2p::PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR + entry.candidates.len() > psy_data::p2p::PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION }) || (response.status == psy_data::p2p::ProposalLookupStatus::Candidates && !has_candidates) || (response.status == psy_data::p2p::ProposalLookupStatus::Empty && has_candidates) { return Err(invalid_data("invalid ProposalLookup response")); } let mut expected = 5; - for entry in response.entries.iter().take(PROPOSAL_LOOKUP_MAX_PAIRS) { + for entry in response.entries.iter().take(PROPOSAL_LOOKUP_MAX_TRANSITIONS) { let entry_bytes = 65 + entry.candidates.len() * psy_data::p2p::PROPOSAL_WIRE_BYTES; if expected + entry_bytes > MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { break; diff --git a/psy_node_common/src/realm/processor/catchup.rs b/psy_node_common/src/realm/processor/catchup.rs index 4b8d9aca..63ba28ec 100644 --- a/psy_node_common/src/realm/processor/catchup.rs +++ b/psy_node_common/src/realm/processor/catchup.rs @@ -7,14 +7,14 @@ use parth_core::protocol::core_types::Q256BitHash; use psy_data::p2p::{ sha256, BodyChunkRequest, BodyChunkResponse, NodeId, Proposal, ProposalLookupEntry, ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, - BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, PROPOSAL_LOOKUP_CONCURRENCY, - PROPOSAL_LOOKUP_ROUND_SECS, PROPOSAL_LOOKUP_TIMEOUT_SECS, PROPOSAL_LOOKUP_WINDOW_PAIRS, + BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, PROPOSAL_LOOKUP_CONCURRENCY, + PROPOSAL_LOOKUP_ROUND_SECS, PROPOSAL_LOOKUP_TIMEOUT_SECS, PROPOSAL_LOOKUP_WINDOW_TRANSITIONS, }; use crate::realm::network::RealmNetworkCommands; -use crate::realm::processor::proposal_store::{ProposalStore, StagedProposal}; +use crate::realm::processor::proposal_backup::{ProposalBackup, StagedProposal}; -pub const CATCHUP_PAIR_ATTEMPTS: usize = 3; +pub const CATCHUP_TRANSITION_ATTEMPTS: usize = 3; /// One window transition after the fetch stage: staged bytes awaiting verification. pub enum TransitionFetchOutcome { @@ -27,7 +27,7 @@ pub enum TransitionFetchOutcome { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum UnappliedTransition { None { accounted_checkpoint: u64 }, - Real { pair: RealmTransition, included_checkpoint: u64 }, + Real { transition: RealmTransition, included_checkpoint: u64 }, } /// Last-modified events must be chronological. Equal roots are leaf rewrites, not bodies. @@ -45,7 +45,7 @@ pub(crate) fn first_unapplied_transition( } if new_root != old_root { return UnappliedTransition::Real { - pair: RealmTransition { old_root, new_root }, + transition: RealmTransition { old_root, new_root }, included_checkpoint: checkpoint_id, }; } @@ -57,7 +57,7 @@ pub(crate) fn first_unapplied_transition( /// Peer set chosen once per catch-up batch: one primary and at most one backup. /// -/// The lookup key is the proposal's (old_root, new_root) pair, so any peer that +/// The lookup key is the proposal's (old_root, new_root) transition, so any peer that /// stored the proposal can answer; peers are tried in ascending validator sub-id /// order and never rescanned per item. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -87,11 +87,11 @@ impl CatchupPeers { } } -/// Stage one proposal per pair: the primary peer answers the whole window, the +/// Stage one proposal per transition: the primary peer answers the whole window, the /// backup answers only when the primary fails, and never more than one backup. pub async fn stage_transition_blocks( client: &RealmNetworkCommands, - store: &ProposalStore, + proposal_backup: &ProposalBackup, peers: &CatchupPeers, chain_id: u64, realm_id: u32, @@ -99,7 +99,7 @@ pub async fn stage_transition_blocks( rejected_proposal_ids: &[[u8; 32]], ) -> Vec { let mut staged = Vec::with_capacity(needed.len()); - let mut windows = needed.chunks(PROPOSAL_LOOKUP_WINDOW_PAIRS); + let mut windows = needed.chunks(PROPOSAL_LOOKUP_WINDOW_TRANSITIONS); let mut tasks = futures::stream::FuturesUnordered::new(); while tasks.len() < PROPOSAL_LOOKUP_CONCURRENCY { let Some(window) = windows.next() else { @@ -107,7 +107,7 @@ pub async fn stage_transition_blocks( }; tasks.push(stage_transition_window( client, - store, + proposal_backup, peers, chain_id, realm_id, @@ -120,7 +120,7 @@ pub async fn stage_transition_blocks( if let Some(window) = windows.next() { tasks.push(stage_transition_window( client, - store, + proposal_backup, peers, chain_id, realm_id, @@ -134,7 +134,7 @@ pub async fn stage_transition_blocks( async fn stage_transition_window( client: &RealmNetworkCommands, - store: &ProposalStore, + proposal_backup: &ProposalBackup, peers: &CatchupPeers, chain_id: u64, realm_id: u32, @@ -169,7 +169,7 @@ async fn stage_transition_window( let (head, tail) = window.split_at(window.len() / 2); let mut staged = Box::pin(stage_transition_window( client, - store, + proposal_backup, peers, chain_id, realm_id, @@ -180,7 +180,7 @@ async fn stage_transition_window( staged.extend( Box::pin(stage_transition_window( client, - store, + proposal_backup, peers, chain_id, realm_id, @@ -207,7 +207,7 @@ async fn stage_transition_window( continue; }; match download_proposal_body(client, peer, &candidate, deadline).await { - Ok(body) => match store.create_staged(&candidate, &body).await { + Ok(body) => match proposal_backup.create_staged(&candidate, &body).await { Ok(staged_proposal) => staged.push(TransitionFetchOutcome::Staged(*transition, staged_proposal)), Err(error) => staged.push(TransitionFetchOutcome::Failed(*transition, error)), }, @@ -228,7 +228,7 @@ async fn lookup_pending_transitions( let request = ProposalLookupRequest { chain_id, realm_id, - pairs: window.to_vec(), + transitions: window.to_vec(), }; let mut last_error = None; for peer in [Some(peers.primary), peers.backup].into_iter().flatten() { @@ -236,7 +236,7 @@ async fn lookup_pending_transitions( Ok(response) => return Ok((peer, response)), Err(error) => { tracing::warn!( - "catch-up window lookup peer={peer} pairs={} error={error:#}", + "catch-up window lookup peer={peer} transitions={} error={error:#}", window.len() ); last_error = Some(error); @@ -328,8 +328,15 @@ async fn download_proposal_body( #[cfg(test)] mod tests { - use super::{first_unapplied_transition, UnappliedTransition}; - use psy_data::p2p::RealmTransition; + + use super::*; + use crate::realm::processor::proposal_backup::ProposalBackup; + use psy_config::CHECKPOINTS_PER_EPOCH; + use psy_data::p2p::{ + NodeId, Proposal, ProposalLookupEntry, ProposalLookupRequest, ProposalLookupResponse, + ProposalLookupStatus, RealmTransition, PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION, + PROPOSAL_LOOKUP_CONCURRENCY, + }; const ROOT_A: [u8; 32] = [0xA; 32]; const ROOT_B: [u8; 32] = [0xB; 32]; @@ -351,7 +358,7 @@ mod tests { assert_eq!( outcome, UnappliedTransition::Real { - pair: RealmTransition { + transition: RealmTransition { old_root: ROOT_A, new_root: ROOT_B, }, @@ -366,7 +373,7 @@ mod tests { assert_eq!( outcome, UnappliedTransition::Real { - pair: RealmTransition { + transition: RealmTransition { old_root: ROOT_A, new_root: ROOT_B, }, @@ -385,4 +392,429 @@ mod tests { } ); } + + fn build_proposal_with_body(old_root: [u8; 32], new_root: [u8; 32], salt: u8) -> (psy_data::p2p::Proposal, Vec) { + build_proposal_with_body_at_checkpoint(old_root, new_root, salt, 99, 1) + } + + fn build_proposal_with_body_at_checkpoint( + old_root: [u8; 32], + new_root: [u8; 32], + salt: u8, + base_checkpoint_id: u64, + proposer_sub_id: u16, + ) -> (psy_data::p2p::Proposal, Vec) { + let output = vec![salt; psy_data::p2p::MAX_FINALIZER_OUTPUT_BYTES]; + let proof = vec![0xABu8; 32]; + let mut state_updates = vec![0u8; 40 + 64 + 20]; + state_updates[40..72].copy_from_slice(&old_root); + state_updates[72..104].copy_from_slice(&new_root); + let worker_tag = [0x11u8; 32]; + let body = psy_data::p2p::encode_proposal_body(&output, &proof, &state_updates, &worker_tag).unwrap(); + let proposal = psy_data::p2p::proposal_from_parts( + 1, + 0, + base_checkpoint_id, + proposer_sub_id, + [salt; 32], + psy_data::p2p::sha256(&output), + psy_data::p2p::sha256(&proof), + psy_data::p2p::sha256(&state_updates), + psy_data::p2p::sha256(&body), + ); + (proposal, body) + } + + fn test_node(seed: u8) -> NodeId { + let mut raw = [0u8; 38]; + raw[..6].copy_from_slice(&[0x00, 0x24, 0x08, 0x01, 0x12, 0x20]); + raw[6..].fill(seed); + NodeId::from_raw(raw).unwrap() + } + + #[derive(Clone, Copy, Debug)] + enum PeerFault { + Timeout, + NotAValidator(NodeId), + Closed, + } + + impl PeerFault { + fn to_error(self) -> crate::realm::network::NetworkError { + use crate::realm::network::NetworkError; + match self { + PeerFault::Timeout => NetworkError::Timeout("test fault".to_string()), + PeerFault::NotAValidator(node) => NetworkError::NotAValidator(node), + PeerFault::Closed => NetworkError::CommandChannelClosed, + } + } + } + + + #[derive(Default)] + struct PeerDoubleState { + offers: std::collections::HashMap>, + bodies: std::collections::HashMap<(NodeId, [u8; 32]), Result, PeerFault>>, + faults: std::collections::HashMap, + looked_up: tokio::sync::Mutex>, + inflight: std::sync::atomic::AtomicUsize, + max_inflight: std::sync::atomic::AtomicUsize, + } + + impl PeerDoubleState { + fn offer(mut self, peer: NodeId, lookup: RealmTransition, proposal: Proposal) -> Self { + self.offers.entry(peer).or_default().push((lookup, proposal)); + self + } + + fn body(mut self, peer: NodeId, proposal_id: [u8; 32], body: Result, PeerFault>) -> Self { + self.bodies.insert((peer, proposal_id), body); + self + } + + fn failing(mut self, peer: NodeId, fault: PeerFault) -> Self { + self.faults.insert(peer, fault); + self + } + } + + /// A command channel answering lookups and body ranges from a fixed inventory. + fn spawn_peer_double(state: std::sync::Arc) -> RealmNetworkCommands { + use crate::realm::network::{NetworkError, RealmNetworkCommand}; + let (commands, mut rx) = tokio::sync::mpsc::channel(64); + tokio::spawn(async move { + while let Some(command) = rx.recv().await { + match command { + RealmNetworkCommand::LookupProposal { + destination, + request, + response, + } => { + let live = state + .inflight + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + + 1; + state + .max_inflight + .fetch_max(live, std::sync::atomic::Ordering::SeqCst); + let answered = match state.faults.get(&destination) { + Some(fault) => Err(fault.to_error()), + None => { + let mut looked = state.looked_up.lock().await; + let offers = state.offers.get(&destination); + let mut entries = Vec::with_capacity(request.transitions.len()); + for transition in &request.transitions { + looked.insert((transition.old_root, transition.new_root)); + let candidates = offers + .map(|offers| { + offers + .iter() + .filter(|(lookup, _)| *lookup == *transition) + .map(|(_, proposal)| proposal.clone()) + .take(PROPOSAL_LOOKUP_CANDIDATES_PER_TRANSITION) + .collect() + }) + .unwrap_or_default(); + entries.push(ProposalLookupEntry { + transition: *transition, + candidates, + }); + } + Ok(ProposalLookupResponse::candidates(entries)) + } + }; + let _ = response.send(answered); + state + .inflight + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + RealmNetworkCommand::RequestBody { + destination, + request, + response, + } => { + let answered = match state.bodies.get(&(destination, request.proposal_id)) { + Some(Ok(body)) => { + let start = request.offset as usize; + let take = (request.max_bytes as usize) + .min(body.len().saturating_sub(start)); + Ok(BodyChunkResponse { + offset: request.offset, + data: body[start..start + take].to_vec(), + eof: start + take == body.len(), + body_len: body.len() as u64, + body_hash: sha256(body), + }) + } + Some(Err(fault)) => Err(fault.to_error()), + None => Err(NetworkError::CommandChannelClosed), + }; + let _ = response.send(answered); + } + _ => {} + } + } + }); + RealmNetworkCommands::from_channel(commands, test_node(200)) + } + + fn lookup_of(old_root: [u8; 32], new_root: [u8; 32]) -> RealmTransition { + RealmTransition { old_root, new_root } + } + + fn single_peer(seed: u8) -> (Vec<(u16, NodeId)>, NodeId) { + let peer = test_node(seed); + (vec![(1, peer)], peer) + } + + async fn install_staged_proposals(proposal_backup: &ProposalBackup, outcomes: Vec) -> usize { + let mut installed_proposal_count = 0usize; + for outcome in outcomes { + if let TransitionFetchOutcome::Staged(_, staged) = outcome { + proposal_backup.install(staged).await.unwrap(); + installed_proposal_count += 1; + } + } + installed_proposal_count + } + + + #[tokio::test] + async fn history_window_stages_each_offered_transition() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (first, first_body) = build_proposal_with_body([1u8; 32], [2u8; 32], 1); + let (second, second_body) = build_proposal_with_body([2u8; 32], [3u8; 32], 2); + let first_lookup = lookup_of([1u8; 32], [2u8; 32]); + let second_lookup = lookup_of([2u8; 32], [3u8; 32]); + let (members, peer) = single_peer(1); + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(peer, first_lookup, first.clone()) + .offer(peer, second_lookup, second.clone()) + .body(peer, first.proposal_id, Ok(first_body)) + .body(peer, second.proposal_id, Ok(second_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &proposal_backup, + &peers, + 1, + 0, + &[first_lookup, second_lookup], + &[], + ) + .await; + assert_eq!(outcomes.len(), 2); + assert!(proposal_backup.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap().is_empty()); + assert_eq!(install_staged_proposals(&proposal_backup, outcomes).await, 2); + assert_eq!( + proposal_backup.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap()[0].proposal_id, + first.proposal_id + ); + assert_eq!( + proposal_backup.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap()[0].proposal_id, + second.proposal_id + ); + } + + #[tokio::test] + async fn history_window_transition_failure_does_not_poison_sibling() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (good, good_body) = build_proposal_with_body([4u8; 32], [5u8; 32], 5); + let (bad, mut bad_body) = build_proposal_with_body([5u8; 32], [6u8; 32], 6); + bad_body[0] ^= 0xFF; + let good_lookup = lookup_of([4u8; 32], [5u8; 32]); + let bad_lookup = lookup_of([5u8; 32], [6u8; 32]); + let (members, peer) = single_peer(3); + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(peer, good_lookup, good.clone()) + .offer(peer, bad_lookup, bad.clone()) + .body(peer, good.proposal_id, Ok(good_body)) + .body(peer, bad.proposal_id, Ok(bad_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &proposal_backup, + &peers, + 1, + 0, + &[good_lookup, bad_lookup], + &[], + ) + .await; + assert!(matches!(outcomes[0], TransitionFetchOutcome::Staged(..))); + assert!(matches!(outcomes[1], TransitionFetchOutcome::Failed(..))); + assert_eq!(install_staged_proposals(&proposal_backup, outcomes).await, 1); + assert_eq!(proposal_backup.lookup_transition(&[4u8; 32], &[5u8; 32]).await.unwrap().len(), 1); + assert!(proposal_backup.lookup_transition(&[5u8; 32], &[6u8; 32]).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn history_lookup_failure_switches_to_backup_peer() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (proposal, body) = build_proposal_with_body([7u8; 32], [8u8; 32], 7); + let lookup = lookup_of([7u8; 32], [8u8; 32]); + let primary = test_node(4); + let backup = test_node(5); + let members = vec![(1, primary), (2, backup)]; + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .failing(primary, PeerFault::Timeout) + .offer(backup, lookup, proposal.clone()) + .body(backup, proposal.proposal_id, Ok(body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks(&client, &proposal_backup, &peers, 1, 0, &[lookup], &[]).await; + assert_eq!(install_staged_proposals(&proposal_backup, outcomes).await, 1); + assert_eq!( + proposal_backup.lookup_transition(&[7u8; 32], &[8u8; 32]).await.unwrap()[0].proposal_id, + proposal.proposal_id + ); + } + + #[tokio::test] + async fn history_window_fetch_concurrency_and_coverage() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (members, peer) = single_peer(8); + let mut state = PeerDoubleState::default(); + let mut needed = Vec::new(); + for i in 0..5u8 { + let old = [i; 32]; + let new = [i + 1; 32]; + let (proposal, mut body) = build_proposal_with_body(old, new, 20 + i); + if i == 2 { + body[0] ^= 0xFF; + } + let lookup = lookup_of(old, new); + state = state + .offer(peer, lookup, proposal.clone()) + .body(peer, proposal.proposal_id, Ok(body)); + needed.push(lookup); + } + let state = std::sync::Arc::new(state); + let client = spawn_peer_double(state.clone()); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks(&client, &proposal_backup, &peers, 1, 0, &needed, &[]).await; + assert_eq!(install_staged_proposals(&proposal_backup, outcomes).await, 4); + assert!( + state.max_inflight.load(std::sync::atomic::Ordering::SeqCst) <= PROPOSAL_LOOKUP_CONCURRENCY + ); + let looked = state.looked_up.lock().await; + for lookup in &needed { + assert!( + looked.contains(&(lookup.old_root, lookup.new_root)), + "missing lookup transition=({},{})", + hex::encode(lookup.old_root), + hex::encode(lookup.new_root) + ); + } + drop(looked); + assert_eq!(proposal_backup.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap().len(), 0); + assert_eq!(proposal_backup.lookup_transition(&[0u8; 32], &[1u8; 32]).await.unwrap().len(), 1); + assert_eq!(proposal_backup.lookup_transition(&[3u8; 32], &[4u8; 32]).await.unwrap().len(), 1); + } + + #[tokio::test] + async fn history_window_across_epochs() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let epoch = CHECKPOINTS_PER_EPOCH; + let first_proposal_count = epoch - 1; + let second_proposal_count = epoch * 3 - 1; + let (first, first_body) = + build_proposal_with_body_at_checkpoint([1u8; 32], [2u8; 32], 30, first_proposal_count, 1); + let (second, second_body) = + build_proposal_with_body_at_checkpoint([2u8; 32], [3u8; 32], 31, second_proposal_count, 2); + let first_lookup = lookup_of([1u8; 32], [2u8; 32]); + let second_lookup = lookup_of([2u8; 32], [3u8; 32]); + let supplier = test_node(9); + let members = vec![(3, supplier)]; + let client = spawn_peer_double(std::sync::Arc::new( + PeerDoubleState::default() + .offer(supplier, first_lookup, first.clone()) + .offer(supplier, second_lookup, second.clone()) + .body(supplier, first.proposal_id, Ok(first_body)) + .body(supplier, second.proposal_id, Ok(second_body)), + )); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let outcomes = stage_transition_blocks( + &client, + &proposal_backup, + &peers, + 1, + 0, + &[first_lookup, second_lookup], + &[], + ) + .await; + assert_eq!(install_staged_proposals(&proposal_backup, outcomes).await, 2); + assert_eq!(first.proposer_sub_id, 1); + assert_eq!(second.proposer_sub_id, 2); + let first_found = proposal_backup.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); + let second_found = proposal_backup.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap(); + assert_eq!(first_found[0].proposal_id, first.proposal_id); + assert_eq!(second_found[0].proposal_id, second.proposal_id); + assert!(first.base_checkpoint_id < second.base_checkpoint_id); + } + + #[tokio::test] + async fn history_window_without_anchor_leaf_fails_closed() { + let error = CatchupPeers::select(&[], 1).expect_err("empty occupied leaves must fail closed"); + assert!( + error + .to_string() + .contains("no other validator peer at this checkpoint"), + "{error}" + ); + } + + #[tokio::test] + async fn history_empty_answer_marks_every_transition_absent() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (members, _peer) = single_peer(10); + let state = std::sync::Arc::new(PeerDoubleState::default()); + let client = spawn_peer_double(state.clone()); + let peers = CatchupPeers::select(&members, 9).unwrap(); + let needed = vec![lookup_of([8u8; 32], [9u8; 32])]; + let outcomes = stage_transition_blocks(&client, &proposal_backup, &peers, 1, 0, &needed, &[]).await; + assert!(matches!(outcomes[0], TransitionFetchOutcome::Absent(..))); + assert!(state.looked_up.lock().await.contains(&([8u8; 32], [9u8; 32]))); + } + + #[tokio::test] + async fn history_resend_revotes() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (proposal, body) = build_proposal_with_body([3u8; 32], [4u8; 32], 3); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + let first = proposal_backup + .read_body_chunk(&BodyChunkRequest { + proposal_id: proposal.proposal_id, + offset: 0, + max_bytes: 64, + }) + .await + .unwrap(); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + let second = proposal_backup + .read_body_chunk(&BodyChunkRequest { + proposal_id: proposal.proposal_id, + offset: 0, + max_bytes: 64, + }) + .await + .unwrap(); + assert_eq!(first.body_hash, proposal.body_hash); + assert_eq!(second.body_hash, proposal.body_hash); + assert_eq!(first.body_len, second.body_len); + } } diff --git a/psy_node_common/src/realm/processor/core/mod.rs b/psy_node_common/src/realm/processor/core/mod.rs index e8b2c939..af2485a0 100644 --- a/psy_node_common/src/realm/processor/core/mod.rs +++ b/psy_node_common/src/realm/processor/core/mod.rs @@ -69,9 +69,9 @@ pub struct PsyRealmProcessor< /// Local validator BLS secret key used to sign the processor's own Vote. /// Required when P2P is enabled; `set_realm_p2p` wires it. pub bls_secret: Option, - pub proposal_store: std::sync::Arc, + pub proposal_backup: std::sync::Arc, pub baseline_replay_rx: Option< - mpsc::Receiver>, + mpsc::Receiver>, >, pub file_system: Arc, pub guta_gatherer_backup_directory: String, diff --git a/psy_node_common/src/realm/processor/core/process_block.rs b/psy_node_common/src/realm/processor/core/process_block.rs index a5504805..3d8ab786 100644 --- a/psy_node_common/src/realm/processor/core/process_block.rs +++ b/psy_node_common/src/realm/processor/core/process_block.rs @@ -235,12 +235,12 @@ where let _ = request.reply.send(result); } self.db.sync_with_coordinator().await?; - self.commit_included_proposal_ffs().await?; + self.apply_proposal_ffs().await?; self.db.ensure_db_matches_coordinator_head().await } - async fn commit_included_proposal_ffs(&mut self) -> anyhow::Result<()> + async fn apply_proposal_ffs(&mut self) -> anyhow::Result<()> where N::HasherBase: MerkleZeroHasher, { @@ -268,7 +268,7 @@ where } last_modifieds.reverse(); let old_root = self.db.state.last_committed_realm_end_root.into_owned_32bytes(); - let (pair, included_checkpoint_id) = match crate::realm::processor::catchup::first_unapplied_transition( + let (transition, included_checkpoint_id) = match crate::realm::processor::catchup::first_unapplied_transition( self.db.state.last_committed_checkpoint_id, old_root, &last_modifieds, @@ -280,8 +280,8 @@ where } return self.db.sync_to_coordinator_checkpoint_id(latest_checkpoint_id).await; } - crate::realm::processor::catchup::UnappliedTransition::Real { pair, included_checkpoint } => { - (pair, included_checkpoint) + crate::realm::processor::catchup::UnappliedTransition::Real { transition, included_checkpoint } => { + (transition, included_checkpoint) } }; coordinator_realm_state = self.db.coordinator_client @@ -290,9 +290,9 @@ where let coordinator_update = self.db.coordinator_client .rc_get_realm_sync_info(included_checkpoint_id, self.db.state.realm_id_u64) .await?; - let included = crate::realm::processor::recovery::CheckpointIdentity { + let included = crate::realm::processor::ffs::CheckpointIdentity { checkpoint_id: included_checkpoint_id, - checkpoint_hash: coordinator_update + checkpoint_leaf_hash: coordinator_update .checkpoint_sync_info .checkpoint_leaf_hash .into_owned_32bytes(), @@ -300,7 +300,7 @@ where let gathering_start = self.db.state.gathering_realm_start_root; let mut selected = match self .db - .verify_history_transition(&included, pair, None, &self.proposal_store) + .verify_history_transition(&included, transition, None, &self.proposal_backup) .await { Ok(Some(verified)) if included.checkpoint_id > self.db.state.last_committed_checkpoint_id => { @@ -312,7 +312,7 @@ where } Ok(Some(verified)) => Some((verified.updates, verified.state_updates)), Ok(None) => None, - Err(error) if crate::realm::processor::recovery::invalid_candidate_id(&error).is_some() => None, + Err(error) if crate::realm::processor::ffs::invalid_candidate_id(&error).is_some() => None, Err(error) => return Err(error), }; if selected.is_none() { @@ -327,42 +327,42 @@ where )?; let staged = crate::realm::processor::catchup::stage_transition_blocks( client, - &self.proposal_store, + &self.proposal_backup, &peers, self.db.state.chain_id, self.db.state.realm_id_u64 as u32, - &[pair], + &[transition], &[], ) .await .into_iter() .find_map(|outcome| match outcome { crate::realm::processor::catchup::TransitionFetchOutcome::Staged(_, staged) => Some(staged), - crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => { + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(transition) => { tracing::debug!( - "no peer offered pair=({},{})", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + "no peer offered transition=({},{})", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ); None } - crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => { + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(transition, error) => { tracing::warn!( - "peer fetch failed pair=({},{}) error={error:#}", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + "peer fetch failed transition=({},{}) error={error:#}", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ); None } }); selected = match self .db - .verify_history_transition(&included, pair, staged.as_ref(), &self.proposal_store) + .verify_history_transition(&included, transition, staged.as_ref(), &self.proposal_backup) .await { Ok(Some(verified)) => { if let Some(staged) = staged { - self.proposal_store.install(staged).await?; + self.proposal_backup.install(staged).await?; } if included.checkpoint_id > self.db.state.last_committed_checkpoint_id { Some( @@ -376,7 +376,7 @@ where } Ok(None) => None, Err(error) => { - if crate::realm::processor::recovery::invalid_candidate_id(&error).is_none() { + if crate::realm::processor::ffs::invalid_candidate_id(&error).is_none() { return Err(error); } None @@ -394,7 +394,7 @@ where }; if included_checkpoint_id > self.db.state.last_committed_checkpoint_id { tracing::info!( - "Committed Realm proposal FFS checkpoint_id={}", + "Applied proposal FFS checkpoint_id={}", included_checkpoint_id ); } @@ -402,7 +402,7 @@ where self.db.state.gathering_realm_start_root = updates.new_realm_root; self.db.shared_state.update_from_core_state(&self.db.state).await?; if gathering_start != updates.old_realm_root && gathering_start != updates.new_realm_root { - self.rebuild_production_gatherer().await?; + self.recreate_guta_gatherer().await?; } else if updates.realm_sub_id != self.db.state.realm_sub_id_u64 { self.guta_queue_gatherer.fast_forward(updates_bytes).await?; } @@ -780,7 +780,7 @@ where .copied() .filter(|(sub_id, _)| *sub_id != local_sub_id) .collect(); - self.proposal_store.save_proposal(&proposal, &body).await?; + self.proposal_backup.save_proposal(&proposal, &body).await?; cmds.publish_proposal(proposal.clone(), body.clone(), remote_bls_keys).await?; cmds.publish_vote(own_vote.clone()).await?; tracing::info!( @@ -846,7 +846,7 @@ where } let certificate = form_certificate(&proposal, &all_votes)?; validate_certificate(&proposal, &certificate, &validator_sub_ids, &leaf_bls_keys)?; - self.proposal_store.save_proposal(&proposal, &body).await?; + self.proposal_backup.save_proposal(&proposal, &body).await?; let signer_ids = all_votes.iter().map(|(sub_id, _)| *sub_id).collect::>(); tracing::info!( "realm P2P certificate formed proposal={} realm={} target={} epoch={} signers={:?} verified_votes={}", diff --git a/psy_node_common/src/realm/processor/core/runner.rs b/psy_node_common/src/realm/processor/core/runner.rs index 2eecac76..1cb9a5c7 100644 --- a/psy_node_common/src/realm/processor/core/runner.rs +++ b/psy_node_common/src/realm/processor/core/runner.rs @@ -160,7 +160,7 @@ where break; } } - processor.abort_production_gatherer().await; + processor.abort_guta_gatherer().await; processor.db.status.mark_stopped(); print_cf_log_indicator("PSY_REALM_PROCESSOR_STOPPED", &format!("R{}_{}", realm_id, realm_sub_id)); diff --git a/psy_node_common/src/realm/processor/core/startup.rs b/psy_node_common/src/realm/processor/core/startup.rs index 747e8972..73f3e545 100644 --- a/psy_node_common/src/realm/processor/core/startup.rs +++ b/psy_node_common/src/realm/processor/core/startup.rs @@ -57,7 +57,7 @@ where genesis_block_update: PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, file_system: Arc, guta_gatherer_backup_directory: String, - proposal_store: std::sync::Arc, + proposal_backup: std::sync::Arc, proposal_fetch: Option, ) -> anyhow::Result { tracing::info!("[REALM_STARTUP] processor new start"); @@ -72,7 +72,7 @@ where &guta_gatherer_backup_directory, genesis_block_update, &mut global_user_tree, - proposal_store.as_ref(), + proposal_backup.as_ref(), proposal_fetch.as_ref(), ) .await?; @@ -127,7 +127,7 @@ where p2p: None, rotation: None, bls_secret: None, - proposal_store, + proposal_backup, baseline_replay_rx: None, file_system, guta_gatherer_backup_directory, @@ -157,13 +157,13 @@ where pub fn set_baseline_replay_rx( &mut self, baseline_replay_rx: tokio::sync::mpsc::Receiver< - crate::realm::processor::recovery::BaselineReplayRequest, + crate::realm::processor::ffs::BaselineReplayRequest, >, ) { self.baseline_replay_rx = Some(baseline_replay_rx); } - pub async fn abort_production_gatherer(&mut self) { + pub async fn abort_guta_gatherer(&mut self) { let Some(handle) = self.guta_gatherer_join.take() else { return; }; @@ -171,10 +171,10 @@ where match handle.await { Ok(Ok(())) => {} Ok(Err(error)) => { - tracing::error!("production gatherer failed during abort: {error:#}"); + tracing::error!("guta gatherer failed during abort: {error:#}"); } Err(join_error) if !join_error.is_cancelled() => { - tracing::error!("production gatherer join error during abort: {join_error}"); + tracing::error!("guta gatherer join error during abort: {join_error}"); } Err(_) => {} } @@ -184,7 +184,7 @@ where where N::HasherBase: parth_core::crypto::hash::traits::MerkleZeroHasher, { - self.abort_production_gatherer().await; + self.abort_guta_gatherer().await; let (mut global_user_tree,) = load_realm_memory_trees_from_db::( &self.db.db, self.db.state.last_committed_checkpoint_id, @@ -198,19 +198,19 @@ where &self.file_system, &self.guta_gatherer_backup_directory, &mut global_user_tree, - self.proposal_store.as_ref(), + self.proposal_backup.as_ref(), self.p2p.as_ref(), ) .await?; self.db.sync_to_coordinator_set_checkpoint_id().await?; - self.rebuild_production_gatherer().await + self.recreate_guta_gatherer().await } - pub async fn rebuild_production_gatherer(&mut self) -> anyhow::Result<()> + pub async fn recreate_guta_gatherer(&mut self) -> anyhow::Result<()> where N::HasherBase: parth_core::crypto::hash::traits::MerkleZeroHasher, { - self.abort_production_gatherer().await; + self.abort_guta_gatherer().await; let (global_user_tree,) = load_realm_memory_trees_from_db::( &self.db.db, self.db.state.last_committed_checkpoint_id, diff --git a/psy_node_common/src/realm/processor/create.rs b/psy_node_common/src/realm/processor/create.rs index aa732b15..8d1444e7 100644 --- a/psy_node_common/src/realm/processor/create.rs +++ b/psy_node_common/src/realm/processor/create.rs @@ -45,7 +45,7 @@ pub async fn create_realm_processor< circuit_fingerprint_config: PsyNodeCircuitFingerprintConfig, coordinator_client: Arc, proof_verifier: Arc, - proposal_store: Arc, + proposal_backup: Arc, proposal_fetch: Option, ) -> anyhow::Result< PsyRealmProcessor< @@ -96,7 +96,7 @@ where genesis, file_system, guta_gatherer_backup_directory, - proposal_store, + proposal_backup, proposal_fetch, ) .await?; diff --git a/psy_node_common/src/realm/processor/db/commit.rs b/psy_node_common/src/realm/processor/db/commit.rs index 43dffb27..3ac71217 100644 --- a/psy_node_common/src/realm/processor/db/commit.rs +++ b/psy_node_common/src/realm/processor/db/commit.rs @@ -120,13 +120,13 @@ where ) .await?; write_checkpoint_state_records::(db, &coordinator_update.checkpoint_sync_info, membership).await?; - let imt_managed = if checkpoint_id == 0 { + let changed_leaves_on_imt_indexed_trees = if checkpoint_id == 0 { HashSet::new() } else { - crate::realm::processor::recovery::imt_managed_leaves_from_db::(db, realm_update) + crate::realm::processor::db::load_changed_leaves_on_imt_indexed_trees::(db, realm_update) .await? }; - crate::realm::processor::recovery::require_state_update_record_coverage(realm_update, checkpoint_id, &imt_managed)?; + crate::realm::processor::db::require_state_update_record_coverage(realm_update, checkpoint_id, &changed_leaves_on_imt_indexed_trees)?; apply_realm_ffs_updates::(db, checkpoint_id, realm_update).await?; let durable_tip = db.get_latest_checkpoint_id().await?; if checkpoint_id >= durable_tip { diff --git a/psy_node_common/src/realm/processor/db/genesis.rs b/psy_node_common/src/realm/processor/db/genesis.rs index ac0458d7..cd0a0577 100644 --- a/psy_node_common/src/realm/processor/db/genesis.rs +++ b/psy_node_common/src/realm/processor/db/genesis.rs @@ -1,3 +1,5 @@ +//! Genesis bootstrap classification and checkpoint-0 write/seed helpers. + use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; use parth_core::{ QCoreProcCheckpointUniqueId, diff --git a/psy_node_common/src/realm/processor/db/init.rs b/psy_node_common/src/realm/processor/db/init.rs index 5ce250c2..43132908 100644 --- a/psy_node_common/src/realm/processor/db/init.rs +++ b/psy_node_common/src/realm/processor/db/init.rs @@ -515,8 +515,8 @@ impl< where N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, { - /// Root pairs for the window: each realm transition at C consumed the previous - /// transition's root, so the pair chain starts at the committed realm end root. + /// Root transitions for the window: each realm transition at C consumed the previous + /// transition's root, so the chain starts at the committed realm end root. async fn pending_transition_lookups( &self, from_checkpoint: u64, @@ -550,10 +550,10 @@ where &self, from_checkpoint: u64, target_tip: u64, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, client: &crate::realm::network::RealmNetworkCommands, validator_nodes: &[(u16, psy_data::p2p::NodeId)], - ) -> anyhow::Result> + ) -> anyhow::Result> where N::QHash: Q256BitHash, { @@ -566,7 +566,7 @@ where )?; let outcomes = crate::realm::processor::catchup::stage_transition_blocks( client, - proposal_store, + proposal_backup, &peers, self.state.chain_id, self.state.realm_id_u64 as u32, @@ -577,18 +577,18 @@ where let mut staged = HashMap::new(); for outcome in outcomes { match outcome { - crate::realm::processor::catchup::TransitionFetchOutcome::Staged(pair, staged_proposal) => { - staged.insert(pair, staged_proposal); + crate::realm::processor::catchup::TransitionFetchOutcome::Staged(transition, staged_proposal) => { + staged.insert(transition, staged_proposal); } - crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => tracing::debug!( - "catch-up window pair=({},{}) not offered by the batch peer", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(transition) => tracing::debug!( + "catch-up window transition=({},{}) not offered by the batch peer", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ), - crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => tracing::warn!( - "catch-up window pair=({},{}) failed error={error:#}", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(transition, error) => tracing::warn!( + "catch-up window transition=({},{}) failed error={error:#}", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ), } } @@ -597,12 +597,12 @@ where async fn stage_single_transition( &self, - pair: psy_data::p2p::RealmTransition, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + transition: psy_data::p2p::RealmTransition, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, client: &crate::realm::network::RealmNetworkCommands, batch_base: u64, rejected: &[[u8; 32]], - ) -> anyhow::Result> { + ) -> anyhow::Result> { let validator_nodes = self.validator_nodes_at(batch_base).await?; let peers = crate::realm::processor::catchup::CatchupPeers::select( &validator_nodes, @@ -610,51 +610,51 @@ where )?; let outcomes = crate::realm::processor::catchup::stage_transition_blocks( client, - proposal_store, + proposal_backup, &peers, self.state.chain_id, self.state.realm_id_u64 as u32, - &[pair], + &[transition], rejected, ) .await; Ok(outcomes.into_iter().find_map(|outcome| match outcome { crate::realm::processor::catchup::TransitionFetchOutcome::Staged(_, staged) => Some(staged), - crate::realm::processor::catchup::TransitionFetchOutcome::Absent(pair) => { + crate::realm::processor::catchup::TransitionFetchOutcome::Absent(transition) => { tracing::debug!( - "catch-up pair=({},{}) not offered by any batch peer", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + "catch-up transition=({},{}) not offered by any batch peer", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ); None } - crate::realm::processor::catchup::TransitionFetchOutcome::Failed(pair, error) => { + crate::realm::processor::catchup::TransitionFetchOutcome::Failed(transition, error) => { tracing::warn!( - "catch-up pair=({},{}) failed error={error:#}", - hex::encode(pair.old_root), - hex::encode(pair.new_root) + "catch-up transition=({},{}) failed error={error:#}", + hex::encode(transition.old_root), + hex::encode(transition.new_root) ); None } })) } - /// Verify one transition, promote the verified bytes into its record, then apply. + /// Verify one transition, install the verified bytes into its record, then apply. async fn apply_verified_transition( &mut self, - included: &crate::realm::processor::recovery::CheckpointIdentity, - pair: psy_data::p2p::RealmTransition, - staged: Option, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + included: &crate::realm::processor::ffs::CheckpointIdentity, + transition: psy_data::p2p::RealmTransition, + staged: Option, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, rejected: &mut Vec<[u8; 32]>, ) -> anyhow::Result, Vec)>> { let verified = match self - .verify_history_transition(included, pair, staged.as_ref(), proposal_store) + .verify_history_transition(included, transition, staged.as_ref(), proposal_backup) .await { Ok(verified) => verified, Err(error) => { - let Some(proposal_id) = crate::realm::processor::recovery::invalid_candidate_id(&error) else { + let Some(proposal_id) = crate::realm::processor::ffs::invalid_candidate_id(&error) else { return Err(error); }; rejected.push(proposal_id); @@ -665,7 +665,7 @@ where return Ok(None); }; if let Some(staged) = staged { - proposal_store.install(staged).await?; + proposal_backup.install(staged).await?; } Ok(Some( self.apply_history_proposal(included, verified) @@ -724,7 +724,7 @@ where file_system: &FileSystem, guta_gatherer_backup_directory: &str, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, ) -> anyhow::Result<()> { let database_check_state = self.get_database_check_state().await?; @@ -743,7 +743,7 @@ where tracing::warn!("Inconsistent Realm Processor State detected. Initiating Recovery."); self.apply_history_transitions( file_system, guta_gatherer_backup_directory, global_user_tree, - proposal_store, proposal_fetch, database_check_state, target_tip, + proposal_backup, proposal_fetch, database_check_state, target_tip, ).await?; } Ok(()) @@ -780,7 +780,7 @@ where file_system: &FileSystem, guta_gatherer_backup_directory: &str, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, database_check_state: DatabaseCheckState, mut target_tip: u64, @@ -797,13 +797,13 @@ where if let Some(client) = proposal_fetch { let validator_nodes = self.validator_nodes_at(start).await?; staged_transitions = self - .stage_recovery_window(start + 1, target_tip, proposal_store, client, &validator_nodes) + .stage_recovery_window(start + 1, target_tip, proposal_backup, client, &validator_nodes) .await?; self.publish_validator_leaves(proposal_fetch, start).await?; } while checkpoint_id <= target_tip { tracing::info!("Recovering checkpoint {}...", checkpoint_id); - let Some((coordinator_update, pair)) = self.transition_at(checkpoint_id).await? else { + let Some((coordinator_update, transition)) = self.transition_at(checkpoint_id).await? else { // Empty checkpoints still authenticate proof-base roots for a later // included proposal. Persist C only; do not advance the committed marker. self.persist_checkpoint_metadata_range(checkpoint_id, checkpoint_id, start) @@ -821,8 +821,8 @@ where }; if !self.restore_checkpoint_transition( file_system, guta_gatherer_backup_directory, global_user_tree, - proposal_store, proposal_fetch, database_check_state, target_tip, start, - checkpoint_id, &coordinator_update, pair, &mut staged_transitions, + proposal_backup, proposal_fetch, database_check_state, target_tip, start, + checkpoint_id, &coordinator_update, transition, &mut staged_transitions, &mut rejected_proposal_ids, ).await? { continue; @@ -848,18 +848,18 @@ where file_system: &FileSystem, guta_gatherer_backup_directory: &str, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, database_check_state: DatabaseCheckState, target_tip: u64, start: u64, checkpoint_id: u64, coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, - pair: psy_data::p2p::RealmTransition, - staged_transitions: &mut HashMap, + transition: psy_data::p2p::RealmTransition, + staged_transitions: &mut HashMap, rejected_proposal_ids: &mut Vec<[u8; 32]>, ) -> anyhow::Result { - let target_root = N::QHash::from_owned_32bytes(pair.new_root); + let target_root = N::QHash::from_owned_32bytes(transition.new_root); self.state.processing_checkpoint_id = checkpoint_id; self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; let prepared_updates = if checkpoint_id == 0 { @@ -874,7 +874,7 @@ where database_check_state, checkpoint_id, target_tip, target_root, coordinator_update, ).await?; if !recovered_from_backup && !self.retry_history_transition( - checkpoint_id, coordinator_update, pair, proposal_store, proposal_fetch, + checkpoint_id, coordinator_update, transition, proposal_backup, proposal_fetch, start, staged_transitions, rejected_proposal_ids, ).await? { return Ok(false); @@ -1078,34 +1078,34 @@ where &mut self, checkpoint_id: u64, coordinator_update: &psy_data::prepared_block::realm::PsyRealmCoordinatorUpdate, - pair: psy_data::p2p::RealmTransition, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + transition: psy_data::p2p::RealmTransition, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, start: u64, - staged_transitions: &mut HashMap, + staged_transitions: &mut HashMap, rejected_proposal_ids: &mut Vec<[u8; 32]>, ) -> anyhow::Result { - let included = crate::realm::processor::recovery::CheckpointIdentity { + let included = crate::realm::processor::ffs::CheckpointIdentity { checkpoint_id, - checkpoint_hash: coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash.into_owned_32bytes(), + checkpoint_leaf_hash: coordinator_update.checkpoint_sync_info.checkpoint_leaf_hash.into_owned_32bytes(), }; - for attempt in 0..crate::realm::processor::catchup::CATCHUP_PAIR_ATTEMPTS { - let staged = match staged_transitions.remove(&pair) { + for attempt in 0..crate::realm::processor::catchup::CATCHUP_TRANSITION_ATTEMPTS { + let staged = match staged_transitions.remove(&transition) { Some(staged) => Some(staged), None => match proposal_fetch { Some(client) => self.stage_single_transition( - pair, proposal_store, client, start, rejected_proposal_ids, + transition, proposal_backup, client, start, rejected_proposal_ids, ).await?, None => None, }, }; match self.apply_verified_transition( - &included, pair, staged, proposal_store, rejected_proposal_ids, + &included, transition, staged, proposal_backup, rejected_proposal_ids, ).await { Ok(Some(_)) => return Ok(true), Ok(None) => tracing::warn!( - "MissingHistoryProof at C={} attempt={attempt} pair=({},{}) rejected", - checkpoint_id, hex::encode(pair.old_root), hex::encode(pair.new_root) + "MissingHistoryProof at C={} attempt={attempt} transition=({},{}) rejected", + checkpoint_id, hex::encode(transition.old_root), hex::encode(transition.new_root) ), Err(error) => { tracing::warn!( @@ -1120,8 +1120,8 @@ where // round's rejections so the next round re-fetches and re-verifies the same candidate. rejected_proposal_ids.clear(); tracing::warn!( - "MissingHistoryProof at C={}: no verified candidate for pair=({},{}); retrying in 5s", - checkpoint_id, hex::encode(pair.old_root), hex::encode(pair.new_root) + "MissingHistoryProof at C={}: no verified candidate for transition=({},{}); retrying in 5s", + checkpoint_id, hex::encode(transition.old_root), hex::encode(transition.new_root) ); tokio::time::sleep(Duration::from_secs(5)).await; Ok(false) @@ -1144,7 +1144,7 @@ where guta_gatherer_backup_directory: &str, genesis_block_update: PsyPreparedRealmBlockStateUpdatesWithCoordinatorUpdate, global_user_tree: &mut SimpleMemoryMerkleRecorderStore, - proposal_store: &crate::realm::processor::proposal_store::ProposalStore, + proposal_backup: &crate::realm::processor::proposal_backup::ProposalBackup, proposal_fetch: Option<&crate::realm::network::RealmNetworkCommands>, ) -> anyhow::Result<()> { let genesis_checkpoint_root = genesis_block_update.coordinator_update.checkpoint_sync_info.checkpoint_tree_root; @@ -1157,7 +1157,7 @@ where file_system, guta_gatherer_backup_directory, global_user_tree, - proposal_store, + proposal_backup, proposal_fetch, ) .await?; diff --git a/psy_node_common/src/realm/processor/db/mod.rs b/psy_node_common/src/realm/processor/db/mod.rs index 23504b3c..0bcb6c31 100644 --- a/psy_node_common/src/realm/processor/db/mod.rs +++ b/psy_node_common/src/realm/processor/db/mod.rs @@ -1,6 +1,10 @@ mod core; pub use core::*; mod commit; +mod records; +pub(crate) use records::{ + load_changed_leaves_on_imt_indexed_trees, require_state_update_record_coverage, +}; mod genesis; mod init; mod sync; diff --git a/psy_node_common/src/realm/processor/db/records.rs b/psy_node_common/src/realm/processor/db/records.rs new file mode 100644 index 00000000..44716f05 --- /dev/null +++ b/psy_node_common/src/realm/processor/db/records.rs @@ -0,0 +1,511 @@ +//! Commit-path FFS record coverage for contract-state and IMT leaves. + +use std::collections::{HashMap, HashSet}; + +use anyhow::Context; +use parth_core::{ + data::hash::fast_node_serializer::{ + QMerkleStoreFastSingleNodeSerializer, QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, + }, + protocol::core_types::Q256BitHash, +}; +use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::{ + contract::{deserialize_imt_leaf_ffs_entry_v2, IMT_LEAF_FFS_ENTRY_SIZE_V2}, + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, + }, +}; + +use crate::realm::processor::ffs::layout::{ + contract_state_leaves_from_ffs, decode_double_id_node_ffs, require_width, +}; + +pub(crate) fn require_state_update_record_coverage( + updates: &PsyPreparedRealmBlockStateUpdates, + checkpoint_id: u64, + changed_leaves_on_imt_indexed_trees: &HashSet<(u64, u64, u64)>, +) -> anyhow::Result<()> +where + Hash: Copy + Q256BitHash, +{ + require_width( + &updates.update_global_user_tree_nodes_ffs, + QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, + "global user tree FFS", + )?; + require_width( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + "user contract tree FFS", + )?; + require_width( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + "contract state tree FFS", + )?; + require_width( + &updates.update_user_leaves_ffs, + PSY_OBJECT_FFS_SIZE_USER_LEAF, + "user leaf FFS", + )?; + require_width( + &updates.update_contract_state_imt_leaves_ffs, + IMT_LEAF_FFS_ENTRY_SIZE_V2, + "IMT leaf FFS", + )?; + if checkpoint_id == 0 { + return Ok(()); + } + let contract_state_leaves = contract_state_leaves_from_ffs(updates)?; + let mut imt_leaves: HashSet<(u64, u64, u64)> = HashSet::new(); + for chunk in updates + .update_contract_state_imt_leaves_ffs + .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) + { + let (tree_id, tree_sub_id, leaf_index, _, _, _, _, _, _) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + imt_leaves.insert((tree_id, tree_sub_id, leaf_index)); + } + for (user_id, contract_id, index) in changed_leaves_on_imt_indexed_trees { + anyhow::ensure!( + imt_leaves.contains(&(*user_id, *contract_id, *index)), + "InvalidStateUpdates: contract-state leaf user={user_id} contract={contract_id} index={index} has no IMT record" + ); + } + let mut user_contract_leaves: HashSet<(u64, u64)> = HashSet::new(); + if !updates.update_user_contract_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + grouped + .entry(node.key.tree_id) + .or_default() + .push((node.key.level, node.key.index)); + }, + )?; + for (user_id, nodes) in grouped { + let height = nodes.iter().map(|(level, _)| *level).max().unwrap_or(0); + for (level, index) in nodes { + if level == height { + user_contract_leaves.insert((user_id, index)); + } + } + } + } + let contract_pairs: HashSet<(u64, u64)> = contract_state_leaves + .keys() + .map(|(user_id, contract_id, _)| (*user_id, *contract_id)) + .collect(); + for (user_id, contract_id) in &user_contract_leaves { + anyhow::ensure!( + contract_pairs.contains(&(*user_id, *contract_id)), + "InvalidStateUpdates: user-contract leaf user={user_id} contract={contract_id} has no contract-state FFS" + ); + } + Ok(()) +} + +/// Nonempty FFS-changed leaves on IMT-indexed trees. +/// Zeroed leaves omitted. Append index is the previous checkpoint's. +pub(crate) async fn load_changed_leaves_on_imt_indexed_trees( + db: &S, + updates: &PsyPreparedRealmBlockStateUpdates, +) -> anyhow::Result> +where + S: psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader + Sync, + F: parth_core::felt::QFelt64, + Hash: Q256BitHash + Copy + PartialEq, +{ + let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); + let mut changed_trees: HashMap<(u64, u64), Vec> = HashMap::new(); + for ((user_id, contract_id, index), new_value) in contract_state_leaves_from_ffs(updates)? { + if new_value == empty_leaf { + continue; + } + changed_trees + .entry((user_id, contract_id)) + .or_default() + .push(index); + } + let mut changed_leaves_on_imt_indexed_trees = HashSet::new(); + for ((user_id, contract_id), leaves) in changed_trees { + let next_append_index = db + .contract_state_imt_get_next_append_index(user_id, contract_id) + .await + .with_context(|| format!("previous-checkpoint IMT append index read failed user={user_id} contract={contract_id}"))?; + if next_append_index == 0 { + continue; + } + for index in leaves { + changed_leaves_on_imt_indexed_trees.insert((user_id, contract_id, index)); + } + } + Ok(changed_leaves_on_imt_indexed_trees) +} + +#[cfg(test)] +mod tests { + use super::*; + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; + use parth_core::{ + crypto::hash::traits::{MerkleZeroHasher, QFieldHashable}, + data::hash::merkle_node_key::SimpleMerkleNode, + pgoldilocks::{PGoldilocksFelt, PGoldilocksHash, PoseidonHasher}, + protocol::core_types::Q256BitHash, + }; + use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::contract::IMTContractStateLeaf, + }; + use parth_core::data::hash::fast_node_serializer::QMerkleStoreFastDoubleNodeSerializer; + use crate::realm::processor::ffs::baseline_replay::replay_state_updates_into_tree; + use std::collections::{HashMap, HashSet}; + + fn empty_updates(old: PGoldilocksHash, new: PGoldilocksHash) -> PsyPreparedRealmBlockStateUpdates { + PsyPreparedRealmBlockStateUpdates { + realm_id: 0, + realm_sub_id: 0, + unique_pending_id: 0, + proc_checkpoint_unique_id: Default::default(), + old_realm_root: old, + new_realm_root: new, + update_global_user_tree_nodes_ffs: vec![], + update_user_contract_tree_nodes_ffs: vec![], + update_contract_state_tree_nodes_ffs: vec![], + update_user_leaves_ffs: vec![], + update_contract_state_imt_leaves_ffs: vec![], + } + } + + #[test] + fn history_ffs_coverage_requires_imt_for_changed_contract_leaf() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut updates = empty_updates(old, old); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + updates.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); + let error = require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect_err("changed contract-state leaf without IMT must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("no IMT record"), "{error}"); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + updates.update_contract_state_imt_leaves_ffs = + psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ) + .to_vec(); + require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect("matching IMT record must close the coverage set"); + require_state_update_record_coverage(&updates, 0, &HashSet::new()) + .expect("genesis may carry contract-state FFS with empty IMT"); + let mut genesis = empty_updates(old, old); + genesis.update_contract_state_tree_nodes_ffs = updates.update_contract_state_tree_nodes_ffs.clone(); + require_state_update_record_coverage(&genesis, 0, &HashSet::new()) + .expect("genesis contract-state leaves with empty IMT are a legal empty IMT"); + let genesis_error = require_state_update_record_coverage(&genesis, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect_err("the same missing IMT must fail after genesis"); + assert!(genesis_error.to_string().contains("no IMT record"), "{genesis_error}"); + require_state_update_record_coverage(&empty_updates(old, old), 1, &HashSet::new()) + .expect("empty contract-state FFS and empty IMT is a legal no-op"); + } + + #[test] + fn history_ffs_user_contract_requires_contract_state() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut updates = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 2, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + updates.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + let error = require_state_update_record_coverage(&updates, 1, &HashSet::new()) + .expect_err("user-contract leaf without contract-state FFS must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("no contract-state FFS"), "{error}"); + require_state_update_record_coverage(&updates, 0, &HashSet::new()) + .expect("genesis may register a user-contract leaf with empty contract-state and IMT"); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + updates.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + updates.update_contract_state_imt_leaves_ffs = + psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ) + .to_vec(); + require_state_update_record_coverage(&updates, 1, &HashSet::new()) + .expect("user-contract plus matching contract-state and IMT must close the coverage set"); + } + + #[test] + fn history_ffs_genesis_skips_imt_coverage() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut registered = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 262144, + level: 8, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + registered.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + require_state_update_record_coverage(®istered, 0, &HashSet::new()) + .expect("genesis contract registration has empty contract-state and IMT"); + let registered_error = require_state_update_record_coverage(®istered, 1, &HashSet::new()) + .expect_err("non-genesis registration without contract-state FFS must fail"); + assert!( + registered_error.to_string().contains("no contract-state FFS"), + "{registered_error}" + ); + let mut empty_value = empty_updates(old, old); + let empty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 262144, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), + }; + empty_value.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&empty_contract_leaf); + require_state_update_record_coverage(&empty_value, 0, &HashSet::new()) + .expect("genesis empty contract-state leaf with empty IMT is trusted setup"); + require_state_update_record_coverage(&empty_value, 1, &HashSet::new()) + .expect("new empty contract-state leaf does not require IMT"); + let mut nonempty = empty_updates(old, old); + let nonempty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 262144, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), + }; + nonempty.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&nonempty_contract_leaf); + require_state_update_record_coverage(&nonempty, 0, &HashSet::new()) + .expect("genesis non-empty contract-state leaf with empty IMT is trusted setup"); + let nonempty_error = require_state_update_record_coverage(&nonempty, 1, &HashSet::from_iter([(262144u64, 0u64, 0u64)])) + .expect_err("updated non-empty contract-state leaf without IMT must fail"); + assert!(nonempty_error.to_string().contains("no IMT record"), "{nonempty_error}"); + replay_state_updates_into_tree::( + &mut tree, &nonempty, 8, 8, 0, 0, &HashSet::new(), + ) + .expect("verify path must honor checkpoint_id=0 and skip IMT pairing"); + let verify_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &nonempty, + 8, + 8, + 0, + 1, + &HashSet::from_iter([(262144u64, 0u64, 0u64)]), + ) + .expect_err("verify path at C=1 must still require IMT for a non-empty leaf"); + assert!(verify_error.to_string().contains("no IMT record"), "{verify_error}"); + let mut poisoned = empty_updates(old, old); + poisoned.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; + let width_error = require_state_update_record_coverage(&poisoned, 0, &HashSet::new()) + .expect_err("genesis still rejects poisoned FFS widths"); + assert!(width_error.to_string().contains("user leaf FFS"), "{width_error}"); + } + + #[test] + fn history_ffs_positional_contract_leaf_needs_no_imt() { + let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); + let mut positional = empty_updates(old, old); + let positional_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1310720, + tree_sub_id: 0, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + positional.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&positional_leaf); + require_state_update_record_coverage(&positional, 1, &HashSet::new()) + .expect("positional non-empty contract-state leaf with no previous IMT entry is exempt"); + let mut cleared = empty_updates(old, old); + let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 2, + }, + value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), + }; + cleared.update_user_contract_tree_nodes_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); + let zero_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: 1, + tree_sub_id: 2, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), + }; + cleared.update_contract_state_tree_nodes_ffs = + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&zero_leaf); + require_state_update_record_coverage(&cleared, 1, &HashSet::new()) + .expect("user-contract pairing survives a contract-state leaf cleared to zero"); + require_state_update_record_coverage(&cleared, 0, &HashSet::from_iter([(1u64, 2u64, 0u64)])) + .expect("genesis skips IMT coverage even with a declared managed set"); + } + + struct IMTPreimageFixture { + next_append: HashMap<(u64, u64), u64>, + } + + #[async_trait::async_trait] + impl psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader for IMTPreimageFixture { + async fn contract_state_imt_get_leaf_preimage( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _leaf_index: u64, + ) -> anyhow::Result>> { + Ok(None) + } + + async fn contract_state_imt_get_leaf_index_for_key( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _key: &PGoldilocksHash, + ) -> anyhow::Result> { + Ok(None) + } + + async fn contract_state_imt_find_predecessor( + &self, + _checkpoint_id: u64, + _user_id: u64, + _contract_id: u64, + _key: &PGoldilocksHash, + ) -> anyhow::Result<(u64, IMTContractStateLeaf)> { + Ok((0, IMTContractStateLeaf::default())) + } + + async fn contract_state_imt_get_next_append_index(&self, user_id: u64, contract_id: u64) -> anyhow::Result { + Ok(self.next_append.get(&(user_id, contract_id)).copied().unwrap_or(0)) + } + } + + + fn contract_state_leaf_ffs(user_id: u64, contract_id: u64, index: u64, value_byte: u8) -> Vec { + QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec( + &parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { + tree_id: user_id, + tree_sub_id: contract_id, + level: 4, + index, + }, + value: PGoldilocksHash::from_owned_32bytes([value_byte; 32]), + }, + ) + } + + #[tokio::test] + async fn history_changed_leaves_on_imt_indexed_trees_follow_append_index() { + let mut imt_indexed_tree = IMTPreimageFixture { next_append: HashMap::new() }; + imt_indexed_tree.next_append.insert((1, 2), 2); + + let mut new_key = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + new_key.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 5, 0x07); + let changed_leaves_on_imt_indexed_trees = load_changed_leaves_on_imt_indexed_trees::<_, PGoldilocksFelt, PGoldilocksHash>(&imt_indexed_tree, &new_key) + .await + .unwrap(); + assert_eq!(changed_leaves_on_imt_indexed_trees, HashSet::from_iter([(1u64, 2u64, 5u64)])); + let error = require_state_update_record_coverage(&new_key, 1, &changed_leaves_on_imt_indexed_trees) + .expect_err("new key on an IMT-indexed tree without IMT record must fail"); + assert!(error.to_string().contains("no IMT record"), "{error}"); + + let mut tracked_changed = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + tracked_changed.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x05); + let tracked_changed_leaves = load_changed_leaves_on_imt_indexed_trees::<_, PGoldilocksFelt, PGoldilocksHash>(&imt_indexed_tree, &tracked_changed) + .await + .unwrap(); + assert_eq!(tracked_changed_leaves, HashSet::from_iter([(1u64, 2u64, 0u64)])); + + let positional_tree = IMTPreimageFixture { next_append: HashMap::new() }; + let positional_changed_leaves = load_changed_leaves_on_imt_indexed_trees::<_, PGoldilocksFelt, PGoldilocksHash>(&positional_tree, &tracked_changed) + .await + .unwrap(); + assert!(positional_changed_leaves.is_empty(), "trees with no IMT entries are positional"); + require_state_update_record_coverage(&tracked_changed, 1, &positional_changed_leaves) + .expect("collector output for a positional leaf must pass coverage"); + + let mut cleared = empty_updates( + PGoldilocksHash::from_owned_32bytes([1u8; 32]), + PGoldilocksHash::from_owned_32bytes([2u8; 32]), + ); + cleared.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x00); + let cleared_leaves = load_changed_leaves_on_imt_indexed_trees::<_, PGoldilocksFelt, PGoldilocksHash>(&imt_indexed_tree, &cleared) + .await + .unwrap(); + assert!(cleared_leaves.is_empty(), "leaf cleared to zero keeps the no-IMT behavior"); + } +} diff --git a/psy_node_common/src/realm/processor/ffs/adopt.rs b/psy_node_common/src/realm/processor/ffs/adopt.rs new file mode 100644 index 00000000..eff07a9a --- /dev/null +++ b/psy_node_common/src/realm/processor/ffs/adopt.rs @@ -0,0 +1,106 @@ +//! Write-side adoption of a verified history candidate. + +use psy_core::job::job_id::{ProvingJobCircuitType, QProvingJobDataID}; +use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdates; +use psy_io::tokio::TokioLikeFileSystem; +use psy_node_core::{ + p2p::traits::realm_coordinantor::RealmCoordinatorClient, + psy_core_db::traits::full::{ + PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, + }, + psy_temp_db::StandardProcessorTempDBStoreBase, + queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, + store::traits::proof_store::QParthProofStore, +}; +use parth_core::{ + crypto::hash::traits::{FieldQHasher, MerkleZeroHasher}, + protocol::core_types::QNetworkTypesConfig, +}; + +use crate::realm::processor::db::PsyRealmDatabaseProcessor; + +use super::{CheckpointIdentity, VerifiedHistoryCandidate}; + +impl< + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, + STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter + + PsyNodeCoreRewardsTagTreeStoreReader + + Send + + Sync, + GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync, + ProofWorkQueue: QStandardWorkerQueuePublisher + Send + Sync, + TempDatabase: StandardProcessorTempDBStoreBase, + ProofStore: QParthProofStore, + FileSystem: TokioLikeFileSystem + Send + Sync + 'static, + CoordinatorClient: RealmCoordinatorClient + Send + Sync, + > + PsyRealmDatabaseProcessor< + N, + S, + STagTreeRewards, + GUTAUpdateQueue, + ProofWorkQueue, + TempDatabase, + ProofStore, + FileSystem, + CoordinatorClient, + > +where + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, +{ + pub async fn ensure_uncommitted_processing_ids(&mut self, checkpoint_id: u64) -> anyhow::Result<()> { + let pending_id = self.state.processing_unique_pending_id; + let mapped_checkpoint = self.db.get_checkpoint_id_for_unique_pending_id(pending_id).await?; + if pending_id != 0 && mapped_checkpoint == Some(checkpoint_id) { + return Ok(()); + } + let (pending_id, proc_checkpoint_unique_id) = + if let Some(ids) = self.db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await? { + ids + } else if pending_id != 0 && mapped_checkpoint.is_none() { + return Ok(()); + } else { + self.db.inc_unique_pending_id(1).await? + }; + self.state.processing_unique_pending_id = pending_id; + self.state.processing_proc_checkpoint_unique_id = proc_checkpoint_unique_id; + self.temp_db + .set_unique_pending_ids(&self.state.realm_identifier, pending_id, proc_checkpoint_unique_id) + .await?; + Ok(()) + } + + pub async fn apply_history_proposal( + &mut self, + included: &CheckpointIdentity, + verified: VerifiedHistoryCandidate, + ) -> anyhow::Result<(PsyPreparedRealmBlockStateUpdates, Vec)> { + let VerifiedHistoryCandidate { + updates, + state_updates, + coordinator_update, + } = verified; + self.ensure_uncommitted_processing_ids(included.checkpoint_id).await?; + self.state.processing_checkpoint_id = included.checkpoint_id; + self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; + self.state.processing_realm_start_root = updates.old_realm_root; + self.state.processing_realm_end_root = updates.new_realm_root; + if self.state.last_committed_checkpoint_id >= included.checkpoint_id { + anyhow::ensure!( + self.state.last_committed_realm_end_root == updates.new_realm_root, + "InvalidStateUpdates at C={}: committed realm root does not match candidate; refusing second FFS", + included.checkpoint_id + ); + } else { + self.commit_state( + &coordinator_update, + &updates, + ProvingJobCircuitType::GUTANoChange, + vec![], + ) + .await?; + } + Ok((updates, state_updates)) + } +} diff --git a/psy_node_common/src/realm/processor/ffs/baseline_replay.rs b/psy_node_common/src/realm/processor/ffs/baseline_replay.rs new file mode 100644 index 00000000..7492e27f --- /dev/null +++ b/psy_node_common/src/realm/processor/ffs/baseline_replay.rs @@ -0,0 +1,424 @@ +//! Coverage checks, tree replay, and previous-checkpoint contract heights. + +use std::collections::{HashMap, HashSet}; + +use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; +use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; +use parth_core::{ + crypto::hash::traits::{FieldQHasher, MerkleZeroHasher, QFieldHashable}, + data::hash::fast_node_serializer::QMerkleStoreFastZeroNodeSerializer, + felt::{FromPrimitiveValuesFelt, ToU64Value}, + protocol::core_types::{Q256BitHash, QFHashBase}, +}; +use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::{ + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, + user::PQEDUserLeaf, + }, +}; +use psy_serialize::PsyCanonicalDatabaseSerializeBaseSingle; + +use super::{gut_local_key, require_imt_leaf_ffs_consistency}; +use crate::realm::processor::db::require_state_update_record_coverage; + +pub(crate) fn replay_state_updates_into_tree( + tree: &mut SimpleMemoryMerkleRecorderStore, + updates: &PsyPreparedRealmBlockStateUpdates, + coordinator_height: u8, + realm_user_tree_height: u8, + realm_id: u64, + checkpoint_id: u64, + changed_leaves_on_imt_indexed_trees: &HashSet<(u64, u64, u64)>, +) -> anyhow::Result<()> +where + F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, + Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, + H: MerkleZeroHasher + FieldQHasher, +{ + require_state_update_record_coverage(updates, checkpoint_id, changed_leaves_on_imt_indexed_trees)?; + anyhow::ensure!( + tree.get_root() == updates.old_realm_root, + "MissingAuthenticatedState: tree root {:?} is not old_realm_root {:?}", + tree.get_root(), + updates.old_realm_root + ); + + let gut_nodes = if updates.update_global_user_tree_nodes_ffs.is_empty() { + Vec::new() + } else { + QMerkleStoreFastZeroNodeSerializer::deserialize_zero_id_nodes_from_slice::( + &updates.update_global_user_tree_nodes_ffs, + ) + }; + let min_user_id = realm_id << realm_user_tree_height; + let mut last_user: HashMap> = HashMap::new(); + for bytes in updates + .update_user_leaves_ffs + .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) + { + let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; + last_user.insert(leaf.user_id.to_u64_value(), leaf); + } + let mut last_leaf: HashMap = HashMap::new(); + for node in &gut_nodes { + let local = gut_local_key(node.key, coordinator_height, realm_id)?; + if local.level == realm_user_tree_height { + last_leaf.insert(local.index, node.value); + } + } + for (index, value) in &last_leaf { + let previous_leaf = tree.get_leaf_value(*index); + if previous_leaf == *value { + continue; + } + let user_id = min_user_id + *index; + let leaf = last_user.get(&user_id).ok_or_else(|| { + anyhow::anyhow!("InvalidStateUpdates: GUT leaf {index} changed without preimage") + })?; + anyhow::ensure!( + leaf.qfhash::() == *value, + "InvalidStateUpdates: user {user_id} preimage does not bind the GUT leaf" + ); + } + for (index, value) in last_leaf { + tree.set_leaf(index, value); + } + + for node in &gut_nodes { + let local = gut_local_key(node.key, coordinator_height, realm_id)?; + anyhow::ensure!( + tree.get_node_value(&local) == node.value, + "InvalidStateUpdates: declared GUT node {:?}={:?} does not match recomputed {:?}", + local, + node.value, + tree.get_node_value(&local) + ); + } + + for (user_id, leaf) in &last_user { + anyhow::ensure!( + *user_id >= min_user_id, + "InvalidStateUpdates: user_id {user_id} is outside realm {realm_id}" + ); + let local_index = user_id - min_user_id; + let expected = leaf.qfhash::(); + anyhow::ensure!( + tree.get_leaf_value(local_index) == expected, + "InvalidStateUpdates: user {user_id} preimage does not bind the recomputed GUT leaf" + ); + } + + require_imt_leaf_ffs_consistency::(&updates.update_contract_state_imt_leaves_ffs)?; + + anyhow::ensure!( + tree.get_root() == updates.new_realm_root, + "InvalidStateUpdates: recomputed root {:?} is not new_realm_root {:?}", + tree.get_root(), + updates.new_realm_root + ); + Ok(()) +} + +pub(super) async fn load_previous_contract_heights( + db: &S, + previous_checkpoint_id: u64, + contract_ids: impl IntoIterator, +) -> anyhow::Result> +where + S: psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader + Sync, + F: Send + Sync, + Hash: Send + Sync, +{ + let mut unique = Vec::new(); + let mut seen = HashSet::new(); + for contract_id in contract_ids { + if seen.insert(contract_id) { + unique.push(contract_id); + } + } + if unique.is_empty() { + return Ok(HashMap::new()); + } + let fetched = db + .get_contract_tree_heights(previous_checkpoint_id, &unique) + .await + .map_err(|error| { + anyhow::anyhow!( + "MissingAuthenticatedState: contract heights unavailable at previous checkpoint {previous_checkpoint_id}: {error}" + ) + })?; + let mut heights = HashMap::with_capacity(unique.len()); + for (i, contract_id) in unique.into_iter().enumerate() { + heights.insert(contract_id, fetched.get(i).copied().unwrap_or(0)); + } + Ok(heights) +} + +pub(super) fn require_previous_contract_height(heights: &HashMap, contract_id: u64) -> anyhow::Result { + let height = heights.get(&contract_id).copied().unwrap_or(0); + anyhow::ensure!( + height > 0, + "MissingAuthenticatedState: contract {contract_id} height is zero" + ); + Ok(height) +} + +#[cfg(test)] +mod tests { + use super::*; + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; + use parth_core::{ + crypto::hash::traits::{MerkleZeroHasher, QFieldHashable}, + data::hash::{ + fast_node_serializer::{ + QMerkleStoreFastSingleNodeSerializer, QMerkleStoreFastZeroNodeSerializer, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + }, + merkle_node_key::{SimpleMerkleNode, SimpleMerkleNodeKey}, + }, + pgoldilocks::{PGoldilocksFelt, PGoldilocksHash, PoseidonHasher}, + protocol::core_types::Q256BitHash, + }; + use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, + }; + use crate::realm::processor::ffs::layout::{ + replay_double_id_nodes_from_leaves, require_width, + }; + use std::collections::{HashMap, HashSet}; + + fn verify_double_id_node_ffs_bytes( + bytes: &[u8], + width: usize, + parse: impl Fn(&[u8]) -> (u64, u64, u8, u64, Hash), + ) -> anyhow::Result<()> + where + H: MerkleZeroHasher, + Hash: Q256BitHash + Copy + PartialEq + Default + std::fmt::Debug, + { + require_width(bytes, width, "tree node FFS")?; + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); + for chunk in bytes.chunks_exact(width) { + let (tree_id, tree_sub_id, level, index, value) = parse(chunk); + grouped.entry((tree_id, tree_sub_id)).or_default().push((level, index, value)); + } + for nodes in grouped.values() { + let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); + let mut tree = SimpleMemoryMerkleRecorderStore::::new(height.max(1)); + replay_double_id_nodes_from_leaves(&mut tree, nodes)?; + } + Ok(()) + } + + fn empty_updates(old: PGoldilocksHash, new: PGoldilocksHash) -> PsyPreparedRealmBlockStateUpdates { + PsyPreparedRealmBlockStateUpdates { + realm_id: 0, + realm_sub_id: 0, + unique_pending_id: 0, + proc_checkpoint_unique_id: Default::default(), + old_realm_root: old, + new_realm_root: new, + update_global_user_tree_nodes_ffs: vec![], + update_user_contract_tree_nodes_ffs: vec![], + update_contract_state_tree_nodes_ffs: vec![], + update_user_leaves_ffs: vec![], + update_contract_state_imt_leaves_ffs: vec![], + } + } + + #[test] + fn history_bad_ffs() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + let poison = SimpleMerkleNode { + key: SimpleMerkleNodeKey { level: 8, index: 0 }, + value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), + }; + updates.update_global_user_tree_nodes_ffs = + QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&poison); + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("poisoned GUT node must fail baseline replay"); + assert!( + error.to_string().contains("InvalidStateUpdates"), + "{error}" + ); + let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 8, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), + }; + let poison_contract = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { + key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { + tree_id: 1, + level: 4, + index: 0, + }, + value: PGoldilocksHash::from_owned_32bytes([0x44; 32]), + }; + let mut contract_ffs = + QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&contract_leaf); + contract_ffs.extend_from_slice( + &QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&poison_contract), + ); + let leaf_poison = SimpleMerkleNode { + key: SimpleMerkleNodeKey { level: 16, index: 0 }, + value: PGoldilocksHash::from_owned_32bytes([0x22; 32]), + }; + let mut leaf_updates = empty_updates(old, old); + leaf_updates.update_global_user_tree_nodes_ffs = + QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&leaf_poison); + let preimage_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &leaf_updates, + 8, + 8, + 0, + 1, + &HashSet::new(), + ) + .expect_err("changed GUT leaf without preimage must fail"); + assert!( + preimage_error.to_string().contains("InvalidStateUpdates"), + "{preimage_error}" + ); + let contract_error = verify_double_id_node_ffs_bytes::( + &contract_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + (node.key.tree_id, 0, node.key.level, node.key.index, node.value) + }, + ) + .expect_err("poisoned contract node must fail baseline replay"); + assert!( + contract_error.to_string().contains("InvalidStateUpdates"), + "{contract_error}" + ); + } + + #[test] + fn history_poison_included_transition() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + updates.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("poisoned user-leaf width must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(!error.to_string().contains("auto")); + } + + struct CountingHeightStore { + heights: HashMap<(u64, u64), u8>, + calls: std::sync::atomic::AtomicUsize, + batches: std::sync::Mutex)>>, + fail: bool, + } + + impl CountingHeightStore { + fn new(heights: HashMap<(u64, u64), u8>) -> Self { + Self { + heights, + calls: std::sync::atomic::AtomicUsize::new(0), + batches: std::sync::Mutex::new(Vec::new()), + fail: false, + } + } + } + + #[async_trait::async_trait] + impl psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader< + PGoldilocksFelt, + PGoldilocksHash, + > for CountingHeightStore + { + async fn get_contract_tree_heights( + &self, + checkpoint_id: u64, + contract_ids: &[u64], + ) -> anyhow::Result> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.batches + .lock() + .expect("height batch log") + .push((checkpoint_id, contract_ids.to_vec())); + if self.fail { + anyhow::bail!("injected height store failure"); + } + Ok(contract_ids + .iter() + .map(|contract_id| self.heights.get(&(checkpoint_id, *contract_id)).copied().unwrap_or(0)) + .collect()) + } + } + + + // Same C+I union the verify path feeds the loader: CST groups (1,7),(2,7),(1,8) + // then IMT finals (1,7,0),(1,9,0) with a duplicate (1,7,0) that or_insert keeps first. + const C_AND_I_IDS: [u64; 5] = [7, 7, 8, 7, 9]; + + + #[tokio::test] + async fn previous_heights_batch_unique_c_and_i_at_historical_checkpoint() { + let previous = 10u64; + let store = CountingHeightStore::new(HashMap::from([ + ((previous, 7), 8), + ((previous, 8), 16), + ((previous, 9), 24), + ((previous + 1, 7), 32), + ])); + let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>( + &store, + previous, + C_AND_I_IDS, + ) + .await + .expect("batch heights"); + assert_eq!(store.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + let batches = store.batches.lock().expect("height batch log"); + assert_eq!(batches.as_slice(), &[(previous, vec![7, 8, 9])]); + drop(batches); + assert_eq!(require_previous_contract_height(&heights, 7).unwrap(), 8); + assert_eq!(require_previous_contract_height(&heights, 8).unwrap(), 16); + assert_eq!(require_previous_contract_height(&heights, 9).unwrap(), 24); + let later = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous + 1, [7]) + .await + .expect("later checkpoint is a different key"); + assert_eq!(require_previous_contract_height(&later, 7).unwrap(), 32); + } + + #[tokio::test] + async fn previous_heights_reject_zero_missing_and_injected_db_error() { + let previous = 10u64; + let store = CountingHeightStore::new(HashMap::from([((previous, 7), 0)])); + let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous, [7, 8]) + .await + .expect("missing maps to zero without a store error"); + let zero = require_previous_contract_height(&heights, 7).expect_err("zero height must reject"); + assert!(zero.to_string().contains("height is zero"), "{zero}"); + let missing = require_previous_contract_height(&heights, 8).expect_err("absent height must reject"); + assert!(missing.to_string().contains("height is zero"), "{missing}"); + let failing = CountingHeightStore { + fail: true, + ..CountingHeightStore::new(HashMap::new()) + }; + let error = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&failing, previous, [7]) + .await + .expect_err("store failure must reject"); + assert!(error.to_string().contains("MissingAuthenticatedState"), "{error}"); + assert!(error.to_string().contains("injected height store failure"), "{error}"); + assert_eq!(failing.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + } +} diff --git a/psy_node_common/src/realm/processor/ffs/layout.rs b/psy_node_common/src/realm/processor/ffs/layout.rs new file mode 100644 index 00000000..4460d5d6 --- /dev/null +++ b/psy_node_common/src/realm/processor/ffs/layout.rs @@ -0,0 +1,405 @@ +//! FFS byte layout, coordinate transforms, and layout invariants. + +use std::collections::{HashMap, HashSet}; + +use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; +use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; +use parth_core::{ + crypto::hash::{ + merkle_proof::MerkleProofCore, + traits::{FieldQHasher, MerkleHasher, MerkleZeroHasher, QFieldHashable}, + }, + data::hash::{ + fast_node_serializer::{ + QMerkleStoreFastDoubleNodeSerializer, QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + }, + merkle_node_key::SimpleMerkleNodeKey, + }, + felt::FromPrimitiveValuesFelt, + protocol::core_types::{Q256BitHash, QFHashBase}, +}; +use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::contract::{ + deserialize_imt_leaf_ffs_entry_v2, IMTContractStateLeaf, IMT_LEAF_FFS_ENTRY_SIZE_V2, + }, +}; + +pub(super) fn gut_local_key( + key: SimpleMerkleNodeKey, + coordinator_height: u8, + realm_id: u64, +) -> anyhow::Result { + anyhow::ensure!( + key.level >= coordinator_height, + "InvalidStateUpdates: GUT node level {} is below coordinator height {}", + key.level, + coordinator_height + ); + let local_level = key.level - coordinator_height; + let expected_realm_id = if local_level >= 64 { + anyhow::ensure!(key.index == 0, "InvalidStateUpdates: GUT node index does not fit local level"); + 0 + } else { + key.index >> local_level + }; + anyhow::ensure!( + expected_realm_id == realm_id, + "InvalidStateUpdates: GUT node realm {expected_realm_id} does not match {realm_id}" + ); + let local_index = if local_level == 0 { + 0 + } else if local_level >= 64 { + key.index + } else { + key.index & ((1u64 << local_level) - 1) + }; + Ok(SimpleMerkleNodeKey { + level: local_level, + index: local_index, + }) +} + +pub(crate) fn require_width(bytes: &[u8], width: usize, what: &str) -> anyhow::Result<()> { + anyhow::ensure!( + bytes.is_empty() || bytes.len() % width == 0, + "InvalidStateUpdates: {what} length {} is not a multiple of {width}", + bytes.len() + ); + Ok(()) +} + +pub(crate) fn decode_double_id_node_ffs( + bytes: &[u8], + width: usize, + mut visit: impl FnMut(&[u8]), +) -> anyhow::Result<()> { + require_width(bytes, width, "tree node FFS")?; + for chunk in bytes.chunks_exact(width) { + visit(chunk); + } + Ok(()) +} + +pub(super) fn seed_tree_from_merkle_proof( + tree: &mut SimpleMemoryMerkleRecorderStore, + proof: &MerkleProofCore, +) -> anyhow::Result<()> +where + H: MerkleZeroHasher + MerkleHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + anyhow::ensure!( + proof.verify::(), + "MissingAuthenticatedState: previous merkle proof does not verify" + ); + let mut key = SimpleMerkleNodeKey::new(tree.get_height(), proof.index); + for sibling in &proof.siblings { + tree.set_node_value(key.sibling(), *sibling); + key = key.parent(); + } + tree.set_leaf(proof.index, proof.value); + Ok(()) +} + +fn require_declared_double_id_nodes_match( + tree: &SimpleMemoryMerkleRecorderStore, + nodes: &[(u8, u64, Hash)], +) -> anyhow::Result<()> +where + H: MerkleZeroHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + for (level, index, value) in nodes { + let key = SimpleMerkleNodeKey { + level: *level, + index: *index, + }; + anyhow::ensure!( + tree.get_node_value(&key) == *value, + "InvalidStateUpdates: declared tree node {:?}={:?} does not match recomputed {:?}", + key, + value, + tree.get_node_value(&key) + ); + } + Ok(()) +} + +pub(super) fn double_id_leaves_at_level( + nodes: &[(u8, u64, Hash)], + height: u8, +) -> HashMap { + let mut last_leaf = HashMap::new(); + for (level, index, value) in nodes { + if *level == height { + last_leaf.insert(*index, *value); + } + } + last_leaf +} + +pub(super) fn replay_double_id_nodes_from_leaves( + tree: &mut SimpleMemoryMerkleRecorderStore, + nodes: &[(u8, u64, Hash)], +) -> anyhow::Result<()> +where + H: MerkleZeroHasher, + Hash: Copy + PartialEq + Default + std::fmt::Debug, +{ + let height = tree.get_height(); + for (index, value) in double_id_leaves_at_level(nodes, height) { + tree.set_leaf(index, value); + } + require_declared_double_id_nodes_match(tree, nodes) +} + +pub(super) fn require_imt_leaf_ffs_consistency(bytes: &[u8]) -> anyhow::Result<()> +where + F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, + Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, + H: MerkleZeroHasher + FieldQHasher, +{ + require_width(bytes, IMT_LEAF_FFS_ENTRY_SIZE_V2, "IMT leaf FFS")?; + let mut first: HashMap<(u64, u64, u64), ([u8; 32], bool)> = HashMap::new(); + let mut first_key_index: HashMap<(u64, u64, [u8; 32]), u64> = HashMap::new(); + let mut first_new_keys: HashSet<(u64, u64, [u8; 32])> = HashSet::new(); + for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { + let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, leaf_value, next_key, next_index, is_new_key) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + let leaf = IMTContractStateLeaf:: { + key: Hash::from_owned_32bytes(leaf_key), + value: Hash::from_owned_32bytes(leaf_value), + next_key: Hash::from_owned_32bytes(next_key), + next_index: F::from_u64_value(next_index), + }; + anyhow::ensure!( + leaf.qfhash::().into_owned_32bytes() == leaf_hash, + "InvalidStateUpdates: IMT leaf preimage does not bind leaf_hash" + ); + let id = (tree_id, tree_sub_id, leaf_index); + let key_id = (tree_id, tree_sub_id, leaf_key); + if let Some(first_index) = first_key_index.get(&key_id) { + if *first_index != leaf_index { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + } else { + first_key_index.insert(key_id, leaf_index); + } + if let Some((first_key, first_new)) = first.get(&id) { + if is_new_key && !first_new_keys.contains(&key_id) { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + if !*first_new && is_new_key { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + if *first_key != leaf_key { + anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); + } + } else { + first.insert(id, (leaf_key, is_new_key)); + if is_new_key { + first_new_keys.insert(key_id); + } + } + } + let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], u64)> = HashMap::new(); + let mut seen_final = HashSet::new(); + for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { + let (tree_id, tree_sub_id, leaf_index, _, leaf_key, _, next_key, next_index, _) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + if seen_final.insert((tree_id, tree_sub_id, leaf_index)) { + finals.insert((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)); + } + } + for ((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)) in &finals { + if *next_index == 0 { + anyhow::ensure!( + *next_key == [0u8; 32], + "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + let Some((successor_key, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) else { + continue; + }; + anyhow::ensure!( + successor_key == next_key, + "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index} next_index={next_index}" + ); + anyhow::ensure!( + successor_key != leaf_key || *next_index == *leaf_index, + "InvalidStateUpdates: IMT successor key collides with source user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } + Ok(()) +} + +/// Changed contract-state leaves keyed by (user, contract, index). +pub(crate) fn contract_state_leaves_from_ffs( + updates: &PsyPreparedRealmBlockStateUpdates, +) -> anyhow::Result> +where + Hash: Copy + Q256BitHash, +{ + let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); + let mut contract_state_leaves: HashMap<(u64, u64, u64), Hash> = HashMap::new(); + if !updates.update_contract_state_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( + chunk, + ); + grouped + .entry((node.key.tree_id, node.key.tree_sub_id)) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + for ((user_id, contract_id), nodes) in grouped { + let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); + for (index, value) in double_id_leaves_at_level(&nodes, height) { + contract_state_leaves.insert((user_id, contract_id, index), value); + } + } + } + Ok(contract_state_leaves) +} + +#[cfg(test)] +mod tests { + use super::*; + use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; + use parth_core::{ + crypto::hash::traits::{MerkleZeroHasher, QFieldHashable}, + data::hash::fast_node_serializer::QMerkleStoreFastZeroNodeSerializer, + felt::FromPrimitiveValuesFelt, + pgoldilocks::{PGoldilocksFelt, PGoldilocksHash, PoseidonHasher}, + protocol::core_types::Q256BitHash, + }; + use psy_data::{ + prepared_block::realm::PsyPreparedRealmBlockStateUpdates, + v1::qdata::contract::IMTContractStateLeaf, + }; + use crate::realm::processor::ffs::baseline_replay::replay_state_updates_into_tree; + use std::collections::{HashMap, HashSet}; + + fn empty_updates(old: PGoldilocksHash, new: PGoldilocksHash) -> PsyPreparedRealmBlockStateUpdates { + PsyPreparedRealmBlockStateUpdates { + realm_id: 0, + realm_sub_id: 0, + unique_pending_id: 0, + proc_checkpoint_unique_id: Default::default(), + old_realm_root: old, + new_realm_root: new, + update_global_user_tree_nodes_ffs: vec![], + update_user_contract_tree_nodes_ffs: vec![], + update_contract_state_tree_nodes_ffs: vec![], + update_user_leaves_ffs: vec![], + update_contract_state_imt_leaves_ffs: vec![], + } + } + + #[test] + fn history_duplicate_imt() { + let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); + let old = tree.get_root(); + let mut updates = empty_updates(old, old); + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); + let first_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let first_hash = first_leaf.qfhash::(); + let first_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ); + let second_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 7, true, + ); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&first_entry); + bytes.extend_from_slice(&second_entry); + updates.update_contract_state_imt_leaves_ffs = bytes; + let error = replay_state_updates_into_tree::( + &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), + ) + .expect_err("conflicting IMT history must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + let moved = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 9, &first_hash, &first_key, &first_value, &first_next_key, 0, false, + ); + let mut moved_bytes = Vec::new(); + moved_bytes.extend_from_slice(&first_entry); + moved_bytes.extend_from_slice(&moved); + let mut moved_updates = empty_updates(old, old); + moved_updates.update_contract_state_imt_leaves_ffs = moved_bytes; + let moved_error = replay_state_updates_into_tree::( + &mut SimpleMemoryMerkleRecorderStore::::new(8), + &moved_updates, + 8, + 8, + 0, + 1, + &HashSet::new(), + ) + .expect_err("same IMT key at a second leaf index must fail"); + assert!(moved_error.to_string().contains("InvalidStateUpdates"), "{moved_error}"); + let successor_key = PGoldilocksHash::from_owned_32bytes([0x44u8; 32]); + let successor_leaf = IMTContractStateLeaf:: { + key: successor_key, + value: first_value, + next_key: first_next_key, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let successor_hash = successor_leaf.qfhash::(); + let successor = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 4, &successor_hash, &successor_key, &first_value, &first_next_key, 0, false, + ); + let mismatched_next = PGoldilocksHash::from_owned_32bytes([0x99u8; 32]); + let mismatched_leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: mismatched_next, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(4), + }; + let mismatched_hash = mismatched_leaf.qfhash::(); + let mismatched = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &mismatched_hash, &first_key, &first_value, &mismatched_next, 4, false, + ); + let mut successor_bytes = Vec::new(); + successor_bytes.extend_from_slice(&mismatched); + successor_bytes.extend_from_slice(&successor); + let successor_error = require_imt_leaf_ffs_consistency::(&successor_bytes) + .expect_err("IMT next_key must match the successor leaf in the same FFS"); + assert!(successor_error.to_string().contains("InvalidStateUpdates"), "{successor_error}"); + } + + #[test] + fn history_imt_terminal_next_key_must_be_zero() { + let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); + let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); + let nonzero_next = PGoldilocksHash::from_owned_32bytes([0x99; 32]); + let leaf = IMTContractStateLeaf:: { + key: first_key, + value: first_value, + next_key: nonzero_next, + next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), + }; + let leaf_hash = leaf.qfhash::(); + let entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( + 1, 0, 3, &leaf_hash, &first_key, &first_value, &nonzero_next, 0, false, + ); + let error = require_imt_leaf_ffs_consistency::(&entry) + .expect_err("terminal IMT next_index=0 with nonzero next_key must fail"); + assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); + assert!(error.to_string().contains("terminal next_key"), "{error}"); + } +} diff --git a/psy_node_common/src/realm/processor/ffs/mod.rs b/psy_node_common/src/realm/processor/ffs/mod.rs new file mode 100644 index 00000000..9ddc2c06 --- /dev/null +++ b/psy_node_common/src/realm/processor/ffs/mod.rs @@ -0,0 +1,167 @@ +//! Historical proposal verification and baseline FFS replay. + +use parth_core::{felt::QFelt64, protocol::core_types::Q256BitHash}; +use psy_data::prepared_block::realm::{PsyPreparedRealmBlockStateUpdates, PsyRealmCoordinatorUpdate}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CheckpointIdentity { + pub checkpoint_id: u64, + pub checkpoint_leaf_hash: [u8; 32], +} + +pub struct BaselineReplayRequest { + pub previous_checkpoint_id: u64, + pub updates: PsyPreparedRealmBlockStateUpdates, + pub reply: tokio::sync::oneshot::Sender>, +} + +pub struct VerifiedHistoryCandidate { + pub updates: PsyPreparedRealmBlockStateUpdates, + pub state_updates: Vec, + pub coordinator_update: PsyRealmCoordinatorUpdate, +} + +fn history_error(kind: &str, checkpoint_id: u64, detail: impl std::fmt::Display) -> anyhow::Error { + anyhow::anyhow!("{kind} at C={checkpoint_id}: {detail}") +} + +/// Why one history transition failed to verify: local material is absent or +/// stale (wait and retry), or the fetched candidate failed validation (prune +/// it and try the next candidate). +#[derive(Debug, thiserror::Error)] +pub(crate) enum RecoveryError { + #[error("{reason}")] + MissingLocalState { + #[source] + reason: anyhow::Error, + }, + #[error("{reason}")] + InvalidCandidate { + proposal_id: [u8; 32], + #[source] + reason: anyhow::Error, + }, +} + +pub(crate) fn invalid_candidate_id(error: &anyhow::Error) -> Option<[u8; 32]> { + match error.downcast_ref::()? { + RecoveryError::InvalidCandidate { proposal_id, .. } => Some(*proposal_id), + RecoveryError::MissingLocalState { .. } => None, + } +} + +fn missing_local_state(checkpoint_id: u64, detail: impl std::fmt::Display) -> RecoveryError { + RecoveryError::MissingLocalState { + reason: history_error("MissingHistoryProof", checkpoint_id, detail), + } +} + +pub(crate) mod layout; +pub(crate) mod baseline_replay; +mod verify; +mod adopt; + +use layout::{ + decode_double_id_node_ffs, double_id_leaves_at_level, gut_local_key, + replay_double_id_nodes_from_leaves, require_imt_leaf_ffs_consistency, require_width, + seed_tree_from_merkle_proof, +}; +use baseline_replay::{ + load_previous_contract_heights, replay_state_updates_into_tree, + require_previous_contract_height, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::realm::processor::proposal_backup::ProposalBackup; + use psy_data::p2p::Proposal; + + fn build_proposal_with_body(old_root: [u8; 32], new_root: [u8; 32], salt: u8) -> (psy_data::p2p::Proposal, Vec) { + build_proposal_with_body_at_checkpoint(old_root, new_root, salt, 99, 1) + } + + fn build_proposal_with_body_at_checkpoint( + old_root: [u8; 32], + new_root: [u8; 32], + salt: u8, + base_checkpoint_id: u64, + proposer_sub_id: u16, + ) -> (psy_data::p2p::Proposal, Vec) { + let output = vec![salt; psy_data::p2p::MAX_FINALIZER_OUTPUT_BYTES]; + let proof = vec![0xABu8; 32]; + let mut state_updates = vec![0u8; 40 + 64 + 20]; + state_updates[40..72].copy_from_slice(&old_root); + state_updates[72..104].copy_from_slice(&new_root); + let worker_tag = [0x11u8; 32]; + let body = psy_data::p2p::encode_proposal_body(&output, &proof, &state_updates, &worker_tag).unwrap(); + let proposal = psy_data::p2p::proposal_from_parts( + 1, + 0, + base_checkpoint_id, + proposer_sub_id, + [salt; 32], + psy_data::p2p::sha256(&output), + psy_data::p2p::sha256(&proof), + psy_data::p2p::sha256(&state_updates), + psy_data::p2p::sha256(&body), + ); + (proposal, body) + } + + #[tokio::test] + async fn history_ready_loads_retained_proposal() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (proposal, body) = build_proposal_with_body([1u8; 32], [2u8; 32], 1); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + let found = proposal_backup.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, proposal.proposal_id); + let loaded = proposal_backup + .load_proposal(&[1u8; 32], &[2u8; 32]) + .await + .unwrap() + .expect("slot holds the retained transition"); + assert_eq!(loaded.0.proposal_id, proposal.proposal_id); + assert_eq!(loaded.1, body); + } + + #[tokio::test] + async fn history_ready_single_body_per_transition() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (first, first_body) = build_proposal_with_body([9u8; 32], [8u8; 32], 3); + let (second, second_body) = build_proposal_with_body([9u8; 32], [8u8; 32], 4); + proposal_backup.save_proposal(&first, &first_body).await.unwrap(); + proposal_backup.save_proposal(&second, &second_body).await.unwrap(); + let found = proposal_backup.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, second.proposal_id); + let found = proposal_backup.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); + assert_eq!(found[0].proposal_id, second.proposal_id); + } + + #[tokio::test] + async fn history_invalid_candidate_ab() { + let dir = tempfile::tempdir().unwrap(); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (candidate_b, body_b) = build_proposal_with_body([10u8; 32], [11u8; 32], 40); + let (mut candidate_x, body_x) = build_proposal_with_body([10u8; 32], [12u8; 32], 41); + candidate_x.base_checkpoint_id = candidate_b.base_checkpoint_id; + proposal_backup.save_proposal(&candidate_b, &body_b).await.unwrap(); + proposal_backup.save_proposal(&candidate_x, &body_x).await.unwrap(); + let found = proposal_backup.lookup_transition(&[10u8; 32], &[12u8; 32]).await.unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].proposal_id, candidate_x.proposal_id); + let sibling = proposal_backup.lookup_transition(&[10u8; 32], &[11u8; 32]).await.unwrap(); + assert_eq!(sibling[0].proposal_id, candidate_b.proposal_id); + let absent = proposal_backup.lookup_transition(&[11u8; 32], &[10u8; 32]).await.unwrap(); + assert!(absent.is_empty()); + assert!(dir + .path() + .join("bodies") + .join(format!("{}_{}", hex::encode([10u8; 32]), hex::encode([11u8; 32]))) + .exists()); + } +} diff --git a/psy_node_common/src/realm/processor/ffs/verify.rs b/psy_node_common/src/realm/processor/ffs/verify.rs new file mode 100644 index 00000000..5ffec68d --- /dev/null +++ b/psy_node_common/src/realm/processor/ffs/verify.rs @@ -0,0 +1,583 @@ +//! Read-only history verification and baseline FFS replay. + +use std::collections::HashMap; + +use anyhow::Context; +use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; +use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; +use parth_core::{ + crypto::hash::traits::{FieldQHasher, MerkleZeroHasher}, + data::hash::fast_node_serializer::{ + QMerkleStoreFastDoubleNodeSerializer, QMerkleStoreFastSingleNodeSerializer, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + }, + felt::ToU64Value, + protocol::core_types::{Q256BitHash, QNetworkTypesConfig}, +}; +use psy_core::job::job_id::QProvingJobDataID; +use psy_data::{ + p2p::{Proposal, RealmTransition}, + prepared_block::realm::{PsyPreparedRealmBlockStateUpdates, PsyRealmCoordinatorUpdate}, + v1::qdata::{ + contract::{deserialize_imt_leaf_ffs_entry_v2, IMT_LEAF_FFS_ENTRY_SIZE_V2}, + ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, + user::PQEDUserLeaf, + }, +}; +use psy_io::tokio::TokioLikeFileSystem; +use psy_node_core::{ + p2p::{traits::realm_coordinantor::RealmCoordinatorClient, validator_lookup::load_realm_validators_from_tree}, + psy_core_db::traits::full::{ + PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, + }, + psy_temp_db::StandardProcessorTempDBStoreBase, + queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, + store::traits::proof_store::QParthProofStore, +}; +use psy_serialize::PsyCanonicalDatabaseSerializeBaseSingle; + +use crate::{ + backup::realm::load_realm_memory_trees_from_db, + realm::processor::{ + consensus::{ + decode_proposal_state_updates, require_declared_roots_match_zk_output, + validator_tree_root_matches_proof_base, verify_proposal_submission, + }, + db::PsyRealmDatabaseProcessor, + proposal_backup::{ProposalBackup, StagedProposal}, + }, +}; + +use super::{ + decode_double_id_node_ffs, double_id_leaves_at_level, history_error, load_previous_contract_heights, + missing_local_state, replay_double_id_nodes_from_leaves, replay_state_updates_into_tree, + require_previous_contract_height, seed_tree_from_merkle_proof, CheckpointIdentity, RecoveryError, + VerifiedHistoryCandidate, +}; + +impl< + N: QNetworkTypesConfig, + S: PsyRealmProcessorStore + Send + Sync, + STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter + + PsyNodeCoreRewardsTagTreeStoreReader + + Send + + Sync, + GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync, + ProofWorkQueue: QStandardWorkerQueuePublisher + Send + Sync, + TempDatabase: StandardProcessorTempDBStoreBase, + ProofStore: QParthProofStore, + FileSystem: TokioLikeFileSystem + Send + Sync + 'static, + CoordinatorClient: RealmCoordinatorClient + Send + Sync, + > + PsyRealmDatabaseProcessor< + N, + S, + STagTreeRewards, + GUTAUpdateQueue, + ProofWorkQueue, + TempDatabase, + ProofStore, + FileSystem, + CoordinatorClient, + > +where + N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, +{ + pub async fn verify_state_updates_from_baseline( + &self, + previous_checkpoint_id: u64, + updates: &PsyPreparedRealmBlockStateUpdates, + ) -> anyhow::Result<()> { + let changed_leaves_on_imt_indexed_trees = crate::realm::processor::db::load_changed_leaves_on_imt_indexed_trees::(self.db.as_ref(), updates) + .await?; + let mut trees = load_realm_memory_trees_from_db::( + &*self.db, + previous_checkpoint_id, + self.state.realm_id_u64, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState at previous checkpoint {previous_checkpoint_id}: {error}"))?; + let mut tree = trees.into_tuple().0; + let checkpoint_id = previous_checkpoint_id.saturating_add(1); + replay_state_updates_into_tree::( + &mut tree, + updates, + N::COORDINATOR_GLOBAL_USER_TREE_HEIGHT, + N::REALM_GLOBAL_USER_TREE_HEIGHT, + self.state.realm_id_u64, + checkpoint_id, + &changed_leaves_on_imt_indexed_trees, + ) + .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}"))?; + self.verify_double_id_trees_from_previous(previous_checkpoint_id, updates) + .await + .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}")) + } + + async fn verify_double_id_trees_from_previous( + &self, + previous_checkpoint_id: u64, + updates: &PsyPreparedRealmBlockStateUpdates, + ) -> anyhow::Result<()> { + let mut last_user: HashMap> = HashMap::new(); + for bytes in updates + .update_user_leaves_ffs + .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) + { + let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; + last_user.insert(leaf.user_id.to_u64_value(), leaf); + } + let mut user_contract_leaves: HashMap<(u64, u64), N::QHash> = HashMap::new(); + if !updates.update_user_contract_tree_nodes_ffs.is_empty() { + let mut grouped: HashMap> = HashMap::new(); + decode_double_id_node_ffs( + &updates.update_user_contract_tree_nodes_ffs, + QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( + chunk, + ); + grouped + .entry(node.key.tree_id) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + for (user_id, nodes) in grouped { + anyhow::ensure!( + nodes.iter().all(|(level, _, _)| *level <= N::GLOBAL_CONTRACT_TREE_HEIGHT), + "InvalidStateUpdates: user-contract node level exceeds tree height" + ); + let mut tree = SimpleMemoryMerkleRecorderStore::::new( + N::GLOBAL_CONTRACT_TREE_HEIGHT, + ); + let leaves = double_id_leaves_at_level(&nodes, N::GLOBAL_CONTRACT_TREE_HEIGHT); + if leaves.is_empty() && !nodes.is_empty() { + anyhow::bail!( + "MissingAuthenticatedState: user-contract internals for user {user_id} have no leaf preimages" + ); + } + for index in leaves.keys() { + let proof = self + .db + .user_contract_tree_get_merkle_proof(previous_checkpoint_id, user_id, *index) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract proof user={user_id} index={index}: {error}"))?; + seed_tree_from_merkle_proof(&mut tree, &proof)?; + } + replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; + for (index, value) in leaves { + user_contract_leaves.insert((user_id, index), value); + } + let new_root = tree.get_root(); + let bound = if let Some(leaf) = last_user.get(&user_id) { + leaf.user_state_tree_root + } else { + self.db + .get_user_leaf(previous_checkpoint_id, user_id) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user {user_id} leaf unavailable: {error}"))? + .user_state_tree_root + }; + anyhow::ensure!( + new_root == bound, + "InvalidStateUpdates: user {user_id} contract tree root does not bind the user leaf" + ); + } + } + let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, N::QHash)>> = HashMap::new(); + if !updates.update_contract_state_tree_nodes_ffs.is_empty() { + decode_double_id_node_ffs( + &updates.update_contract_state_tree_nodes_ffs, + QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, + |chunk| { + let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( + chunk, + ); + grouped + .entry((node.key.tree_id, node.key.tree_sub_id)) + .or_default() + .push((node.key.level, node.key.index, node.value)); + }, + )?; + } + let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)> = HashMap::new(); + if !updates.update_contract_state_imt_leaves_ffs.is_empty() { + for chunk in updates + .update_contract_state_imt_leaves_ffs + .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) + { + let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, _, next_key, next_index, is_new_key) = + deserialize_imt_leaf_ffs_entry_v2(chunk)?; + finals.entry((tree_id, tree_sub_id, leaf_index)).or_insert(( + leaf_hash, + leaf_key, + next_key, + next_index, + is_new_key, + )); + } + } + let heights = load_previous_contract_heights( + self.db.as_ref(), + previous_checkpoint_id, + grouped + .keys() + .map(|(_, contract_id)| *contract_id) + .chain(finals.keys().map(|(_, contract_id, _)| *contract_id)), + ) + .await?; + let mut contract_state_leaves: HashMap<(u64, u64, u64), N::QHash> = HashMap::new(); + for ((user_id, contract_id), nodes) in grouped { + let height = require_previous_contract_height(&heights, contract_id)?; + anyhow::ensure!( + nodes.iter().all(|(level, _, _)| *level <= height), + "InvalidStateUpdates: contract-state node level exceeds authenticated height {height}" + ); + let mut tree = SimpleMemoryMerkleRecorderStore::::new(height); + let leaves = double_id_leaves_at_level(&nodes, height); + if leaves.is_empty() && !nodes.is_empty() { + anyhow::bail!( + "MissingAuthenticatedState: contract-state internals for user={user_id} contract={contract_id} have no leaf preimages" + ); + } + for index in leaves.keys() { + let proof = self + .db + .contract_state_tree_get_merkle_proof( + previous_checkpoint_id, + user_id, + contract_id, + height, + *index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: contract-state proof user={user_id} contract={contract_id} index={index}: {error}"))?; + seed_tree_from_merkle_proof(&mut tree, &proof)?; + } + replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; + for (index, value) in leaves { + contract_state_leaves.insert((user_id, contract_id, index), value); + } + let new_root = tree.get_root(); + let bound = if let Some(leaf) = user_contract_leaves.get(&(user_id, contract_id)) { + *leaf + } else { + self.db + .user_contract_tree_get_leaf_hash(previous_checkpoint_id, user_id, contract_id) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract leaf user={user_id} contract={contract_id}: {error}"))? + }; + anyhow::ensure!( + new_root == bound, + "InvalidStateUpdates: contract-state root does not bind user {user_id} contract {contract_id}" + ); + } + self.verify_imt_from_previous(previous_checkpoint_id, finals, &contract_state_leaves, &heights) + .await + } + + async fn verify_imt_from_previous( + &self, + previous_checkpoint_id: u64, + finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)>, + contract_state_leaves: &HashMap<(u64, u64, u64), N::QHash>, + heights: &HashMap, + ) -> anyhow::Result<()> { + for ((tree_id, tree_sub_id, leaf_index), (leaf_hash, leaf_key, next_key, next_index, is_new_key)) in &finals { + let height = require_previous_contract_height(heights, *tree_sub_id)?; + let expected = if let Some(value) = contract_state_leaves.get(&(*tree_id, *tree_sub_id, *leaf_index)) { + *value + } else { + self.db + .contract_state_tree_get_leaf_hash( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + height, + *leaf_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))? + }; + anyhow::ensure!( + expected.into_owned_32bytes() == *leaf_hash, + "InvalidStateUpdates: IMT leaf_hash does not bind contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + let key = N::QHash::from_owned_32bytes(*leaf_key); + let old_index = self + .db + .contract_state_imt_get_leaf_index_for_key(previous_checkpoint_id, *tree_id, *tree_sub_id, &key) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT key index user={tree_id} contract={tree_sub_id}: {error}"))?; + if *is_new_key { + anyhow::ensure!( + old_index.is_none(), + "InvalidStateUpdates: IMT is_new_key already indexed user={tree_id} contract={tree_sub_id}" + ); + let previous_at_index = self + .db + .contract_state_imt_get_leaf_preimage( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + *leaf_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT previous leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))?; + anyhow::ensure!( + previous_at_index.is_none(), + "InvalidStateUpdates: IMT new key overwrites an authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } else { + let old_index = old_index.ok_or_else(|| { + anyhow::anyhow!( + "InvalidStateUpdates: IMT key is not new but has no authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" + ) + })?; + anyhow::ensure!( + old_index == *leaf_index, + "InvalidStateUpdates: IMT key index moved user={tree_id} contract={tree_sub_id}" + ); + } + if *next_index == 0 { + anyhow::ensure!( + *next_key == [0u8; 32], + "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + if let Some((_, successor_key, _, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) { + anyhow::ensure!( + successor_key == next_key, + "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + continue; + } + let successor = self + .db + .contract_state_imt_get_leaf_preimage( + previous_checkpoint_id, + *tree_id, + *tree_sub_id, + *next_index, + ) + .await + .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT successor user={tree_id} contract={tree_sub_id} next_index={next_index}: {error}"))?; + let successor = successor.ok_or_else(|| { + anyhow::anyhow!("MissingAuthenticatedState: IMT successor missing user={tree_id} contract={tree_sub_id} next_index={next_index}") + })?; + anyhow::ensure!( + successor.key.into_owned_32bytes() == *next_key, + "InvalidStateUpdates: IMT next_key does not match authenticated successor user={tree_id} contract={tree_sub_id} index={leaf_index}" + ); + } + Ok(()) + } + + pub async fn verify_history_proposal( + &self, + included: &CheckpointIdentity, + proposal: &Proposal, + body: &[u8], + ) -> anyhow::Result<( + PsyPreparedRealmBlockStateUpdates, + Vec, + PsyRealmCoordinatorUpdate, + )> { + anyhow::ensure!( + proposal.realm_id == self.state.realm_id_u64 as u32, + "InvalidStateUpdates at C={}: proposal realm mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + proposal.chain_id == self.state.chain_id, + "InvalidStateUpdates at C={}: proposal chain mismatch", + included.checkpoint_id + ); + let coordinator_update = self + .coordinator_client + .rc_get_realm_sync_info(included.checkpoint_id, self.state.realm_id_u64) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("Coordinator C materials unavailable: {error:#}"), + ))?; + anyhow::ensure!( + coordinator_update.checkpoint_sync_info.checkpoint_id == included.checkpoint_id, + "MissingHistoryProof at C={}: coordinator checkpoint id mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + coordinator_update + .checkpoint_sync_info + .checkpoint_leaf_hash + .into_owned_32bytes() + == included.checkpoint_leaf_hash, + "MissingHistoryProof at C={}: coordinator leaf hash does not match included.checkpoint_leaf_hash", + included.checkpoint_id + ); + let authenticated_leaf = self + .checkpoint_tree_backup_manager + .checkpoint_tree + .get_leaf(included.checkpoint_id); + if authenticated_leaf.value.into_owned_32bytes() != included.checkpoint_leaf_hash { + return Err(missing_local_state( + included.checkpoint_id, + "included.checkpoint_leaf_hash does not match the authenticated checkpoint tree leaf", + ) + .into()); + } + let roots = self + .db + .get_checkpoint_global_state_roots(proposal.base_checkpoint_id) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("proof-base P={} roots unavailable: {error:#}", proposal.base_checkpoint_id), + ))?; + anyhow::ensure!( + validator_tree_root_matches_proof_base( + &proposal.validator_tree_root, + &roots.validator_tree_root.into_owned_32bytes(), + ), + "MissingHistoryProof at C={}: proposal.validator_tree_root does not match P={}", + included.checkpoint_id, + proposal.base_checkpoint_id + ); + let (_, _, user_ids, _) = + load_realm_validators_from_tree::( + &*self.db, + self.state.chain_id, + proposal.base_checkpoint_id, + proposal.realm_id, + &roots.validator_tree_root, + ) + .await + .map_err(|error| missing_local_state( + included.checkpoint_id, + format!("proof-base P={} validator tree unavailable: {error:#}", proposal.base_checkpoint_id), + ))?; + let proposer_user_id = user_ids + .iter() + .find(|(sub_id, _)| *sub_id == proposal.proposer_sub_id) + .map(|(_, user_id)| *user_id) + .ok_or_else(|| { + history_error( + "MissingHistoryProof", + included.checkpoint_id, + format!("proposer sub_id {} is not a validator", proposal.proposer_sub_id), + ) + })?; + let decoded = + verify_proposal_submission::(proposal, body, proposer_user_id, self.proof_verifier.as_ref())?; + let updates = decode_proposal_state_updates::(&decoded.state_updates)?; + anyhow::ensure!( + updates.realm_id == proposal.realm_id as u64, + "InvalidStateUpdates at C={}: FFS realm_id {} does not match proposal {}", + included.checkpoint_id, + updates.realm_id, + proposal.realm_id + ); + anyhow::ensure!( + updates.realm_sub_id == proposal.proposer_sub_id as u64, + "InvalidStateUpdates at C={}: FFS realm_sub_id {} does not match proposer {}", + included.checkpoint_id, + updates.realm_sub_id, + proposal.proposer_sub_id + ); + let output = psy_data::guta::realm_finalize::protocol_decode_finalize_output::( + &decoded.output, + )?; + require_declared_roots_match_zk_output(&updates, &output)?; + anyhow::ensure!( + updates.new_realm_root.into_owned_32bytes() != [0u8; 32] + || updates.old_realm_root == updates.new_realm_root, + "InvalidStateUpdates at C={}: empty new root", + included.checkpoint_id + ); + let realm_proof = &coordinator_update.merkle_proof_to_realm_root; + anyhow::ensure!( + realm_proof.verify::(), + "MissingHistoryProof at C={}: realm-root path does not verify", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.index == self.state.realm_id_u64, + "MissingHistoryProof at C={}: realm-root path index mismatch", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.value == updates.new_realm_root, + "MissingHistoryProof at C={}: authenticated realm root does not match proposal new_realm_root", + included.checkpoint_id + ); + anyhow::ensure!( + realm_proof.root == coordinator_update.checkpoint_sync_info.state_roots.user_tree_root, + "MissingHistoryProof at C={}: realm-root path is not bound to C user_tree_root", + included.checkpoint_id + ); + let previous = included.checkpoint_id + .checked_sub(1) + .ok_or_else(|| history_error("MissingHistoryProof", included.checkpoint_id, "C=0 has no predecessor"))?; + self.verify_state_updates_from_baseline(previous, &updates) + .await + .with_context(|| { + format!( + "InvalidStateUpdates at C={} proposal_id={}", + included.checkpoint_id, + hex::encode(proposal.proposal_id) + ) + })?; + Ok((updates, decoded.state_updates, coordinator_update)) + } + + pub async fn verify_history_transition( + &self, + included: &CheckpointIdentity, + transition: RealmTransition, + staged: Option<&StagedProposal>, + proposal_backup: &ProposalBackup, + ) -> anyhow::Result>> { + let loaded = match staged { + Some(staged) => Some(proposal_backup.read_staged(staged).await?), + None => proposal_backup + .load_proposal(&transition.old_root, &transition.new_root) + .await?, + }; + let Some((proposal, body)) = loaded else { + return Ok(None); + }; + match self + .verify_history_proposal(included, &proposal, &body) + .await + { + Ok((updates, state_updates, coordinator_update)) => { + Ok(Some(VerifiedHistoryCandidate { + updates, + state_updates, + coordinator_update, + })) + } + Err(error) => { + if error + .downcast_ref::() + .is_some_and(|classified| matches!(classified, RecoveryError::MissingLocalState { .. })) + { + return Err(error); + } + tracing::warn!( + "history verify rejected C={} transition=({},{}) proposal={} error={error}", + included.checkpoint_id, + hex::encode(transition.old_root), + hex::encode(transition.new_root), + hex::encode(proposal.proposal_id) + ); + Err(anyhow::Error::new(RecoveryError::InvalidCandidate { + proposal_id: proposal.proposal_id, + reason: anyhow::anyhow!("history verify rejected C={}: {error:#}", included.checkpoint_id), + })) + } + } + } +} diff --git a/psy_node_common/src/realm/processor/mod.rs b/psy_node_common/src/realm/processor/mod.rs index f8d95acf..f5398612 100644 --- a/psy_node_common/src/realm/processor/mod.rs +++ b/psy_node_common/src/realm/processor/mod.rs @@ -6,5 +6,5 @@ pub mod processor_shared_status; pub mod gatherers; pub mod create; pub mod coordinator_sync; -pub mod proposal_store; -pub mod recovery; \ No newline at end of file +pub mod proposal_backup; +pub mod ffs; \ No newline at end of file diff --git a/psy_node_common/src/realm/processor/proposal_store.rs b/psy_node_common/src/realm/processor/proposal_backup.rs similarity index 71% rename from psy_node_common/src/realm/processor/proposal_store.rs rename to psy_node_common/src/realm/processor/proposal_backup.rs index 2f31924f..26a7a3f7 100644 --- a/psy_node_common/src/realm/processor/proposal_store.rs +++ b/psy_node_common/src/realm/processor/proposal_backup.rs @@ -6,6 +6,8 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use anyhow::Context; +use parth_core::protocol::core_types::Q256BitHash; +use parth_core::PHash; use psy_data::p2p::{ BodyChunkRequest, BodyChunkResponse, Proposal, ProposalLookupEntry, ProposalLookupRequest, ProposalLookupResponse, ProtocolEncode, BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, @@ -14,9 +16,8 @@ use psy_data::p2p::{ use psy_io::tokio::{TokioFileLike, TokioLikeFileSystem, TokioStdFileSystem}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use crate::realm::processor::consensus::decode_proposal_body; +use crate::realm::processor::consensus::{decode_proposal_body, decode_proposal_state_updates}; -const STATE_UPDATES_ROOTS_OFFSET: usize = 40; #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub(crate) struct RealmTransition { pub from_root: [u8; 32], @@ -31,7 +32,7 @@ struct TransitionRecord { } /// Owns one staged body file. Dropping it removes the staging file through the -/// store's filesystem abstraction, so an abandoned candidate cannot leak. +/// backup's filesystem abstraction, so an abandoned candidate cannot leak. pub struct StagedProposal { path: String, transition: RealmTransition, @@ -47,14 +48,14 @@ impl Drop for StagedProposal { } } -/// In-memory tables over the retained bodies, guarded by the store's single +/// In-memory tables over the retained bodies, guarded by the backup's single /// lock: records keyed by transition plus the O(1) proposal-id lookup index. struct RetainedBodies { by_transition: HashMap>, by_proposal_id: HashMap<[u8; 32], RealmTransition>, } -pub struct ProposalStore { +pub struct ProposalBackup { fs: std::sync::Arc, root: PathBuf, inner: tokio::sync::Mutex>, @@ -62,18 +63,18 @@ pub struct ProposalStore { instance_id: u64, } -impl ProposalStore { +impl ProposalBackup { pub async fn open(root: impl AsRef) -> anyhow::Result { Self::open_with_fs(root, TokioStdFileSystem).await } } -impl ProposalStore { +impl ProposalBackup { pub async fn open_with_fs(root: impl AsRef, fs: F) -> anyhow::Result { let root = root.as_ref().to_path_buf(); fs.file_like_fs_create_dir_all(&path_string(&root.join("bodies"))).await?; let root = if root.is_absolute() { root } else { std::env::current_dir()?.join(root) }; - let store = Self { + let proposal_backup = Self { fs: std::sync::Arc::new(fs), root, inner: tokio::sync::Mutex::new(RetainedBodies { @@ -83,14 +84,14 @@ impl ProposalStore { staged_seq: AtomicU64::new(1), instance_id: std::hash::RandomState::new().build_hasher().finish(), }; - remove_staged_files(&store.root.join("bodies")).await?; + remove_staged_files(&proposal_backup.root.join("bodies")).await?; { - let mut inner = store.inner.lock().await; - for transition in store.read_stored_transitions().await? { - let _ = store.load_transition_record(&mut inner, &transition).await; + let mut inner = proposal_backup.inner.lock().await; + for transition in proposal_backup.load_retained_transitions().await? { + let _ = proposal_backup.load_transition_record(&mut inner, &transition).await; } } - Ok(store) + Ok(proposal_backup) } // Trusted local consensus/replay writes bypass fetched-body isolation staging verification. @@ -131,19 +132,19 @@ impl ProposalStore { pub async fn lookup_proposal(&self, request: &ProposalLookupRequest) -> anyhow::Result { let mut entries = Vec::new(); let mut wire_bytes = 5; - for pair in &request.pairs { - let candidates = self.lookup_transition(&pair.old_root, &pair.new_root).await? + for transition in &request.transitions { + let candidates = self.lookup_transition(&transition.old_root, &transition.new_root).await? .into_iter().filter(|proposal| proposal.chain_id == request.chain_id && proposal.realm_id == request.realm_id).collect::>(); wire_bytes += 65 + candidates.len() * PROPOSAL_WIRE_BYTES; if wire_bytes > psy_data::p2p::MAX_PROPOSAL_LOOKUP_RESPONSE_BYTES { return Ok(ProposalLookupResponse::truncated(entries)); } - entries.push(ProposalLookupEntry { transition: *pair, candidates }); + entries.push(ProposalLookupEntry { transition: *transition, candidates }); } Ok(ProposalLookupResponse::candidates(entries)) } - /// The store retains one body per transition, so this yields zero or one + /// The backup retains one body per transition, so this yields zero or one /// proposal even though the wire answer carries room for two candidates. /// Cache hits skip disk reads; later on-disk corruption surfaces only on /// the verified load paths (load_proposal / read_body_chunk I/O). @@ -237,7 +238,7 @@ impl ProposalStore { result } - /// A transition whose bytes do not verify is dropped entirely: the pair goes + /// A transition whose bytes do not verify is dropped entirely: the transition goes /// back to absent so an honest body can be staged and installed again. async fn remove_transition(&self, inner: &mut RetainedBodies, transition: &RealmTransition) { if let Some(record) = inner.by_transition.remove(transition) { @@ -247,7 +248,7 @@ impl ProposalStore { if let Err(error) = self.fs.file_like_remove_file(&path).await { if error.kind() != std::io::ErrorKind::NotFound { tracing::warn!( - "failed to remove unverified transition pair=({},{}) error={error}", + "failed to remove unverified transition=({},{}) error={error}", hex::encode(transition.from_root), hex::encode(transition.to_root) ); @@ -255,7 +256,7 @@ impl ProposalStore { } } - /// Verifies the staged bytes and installs them as the pair's retained body. + /// Verifies the staged bytes and installs them as the transition's retained body. pub async fn install(&self, staged: StagedProposal) -> anyhow::Result<()> { let mut inner = self.inner.lock().await; let mut file = self.fs.file_like_fs_open(&staged.path).await?; @@ -288,7 +289,7 @@ impl ProposalStore { Ok(()) } - async fn read_stored_transitions(&self) -> anyhow::Result> { + async fn load_retained_transitions(&self) -> anyhow::Result> { let mut transitions = Vec::new(); let mut dir = tokio::fs::read_dir(self.root.join("bodies")).await?; while let Some(entry) = dir.next_entry().await? { @@ -363,12 +364,11 @@ fn verify_complete_object(proposal: &Proposal, body: &[u8]) -> anyhow::Result anyhow::Result { - anyhow::ensure!(state_updates.len() >= STATE_UPDATES_ROOTS_OFFSET + 64, "state_updates missing old/new realm roots"); - let mut old_root = [0; 32]; - let mut new_root = [0; 32]; - old_root.copy_from_slice(&state_updates[STATE_UPDATES_ROOTS_OFFSET..STATE_UPDATES_ROOTS_OFFSET + 32]); - new_root.copy_from_slice(&state_updates[STATE_UPDATES_ROOTS_OFFSET + 32..STATE_UPDATES_ROOTS_OFFSET + 64]); - Ok(RealmTransition { from_root: old_root, to_root: new_root }) + let updates = decode_proposal_state_updates::(state_updates)?; + Ok(RealmTransition { + from_root: updates.old_realm_root.into_owned_32bytes(), + to_root: updates.new_realm_root.into_owned_32bytes(), + }) } async fn remove_staged_files(dir: &Path) -> anyhow::Result<()> { @@ -400,107 +400,136 @@ fn parse_transition_file_name(name: &str) -> Option { mod tests { use super::*; use psy_data::p2p::{encode_proposal_body, proposal_from_parts, sha256, MAX_FINALIZER_OUTPUT_BYTES}; + use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdates; + use psy_serialize::PsyCanonicalDatabaseSerializeBaseSingle; + + fn canonical_empty_state_updates(from_root: [u8; 32], to_root: [u8; 32]) -> Vec { + PsyPreparedRealmBlockStateUpdates:: { + realm_id: 0, + realm_sub_id: 0, + unique_pending_id: 0, + proc_checkpoint_unique_id: 0, + old_realm_root: PHash::from_owned_32bytes(from_root), + new_realm_root: PHash::from_owned_32bytes(to_root), + update_global_user_tree_nodes_ffs: Vec::new(), + update_user_contract_tree_nodes_ffs: Vec::new(), + update_contract_state_tree_nodes_ffs: Vec::new(), + update_user_leaves_ffs: Vec::new(), + update_contract_state_imt_leaves_ffs: Vec::new(), + } + .psy_ser_to_bytes_vec() + .unwrap() + } - fn sample_object(salt: u8) -> (Proposal, Vec) { + fn build_proposal_with_body(salt: u8) -> (Proposal, Vec) { let output = vec![salt; MAX_FINALIZER_OUTPUT_BYTES]; let proof = vec![0xAB; 32]; - let mut updates = vec![0; STATE_UPDATES_ROOTS_OFFSET + 64 + 20]; - updates[40..72].fill(1); - updates[72..104].fill(2); + let updates = canonical_empty_state_updates([1; 32], [2; 32]); let body = encode_proposal_body(&output, &proof, &updates, &[0x11; 32]).unwrap(); let proposal = proposal_from_parts(1, 0, 99, 1, [salt; 32], sha256(&output), sha256(&proof), sha256(&updates), sha256(&body)); (proposal, body) } + #[test] + fn typed_roots_match_canonical_header_bytes() { + let updates = canonical_empty_state_updates([1; 32], [2; 32]); + assert_eq!(&updates[40..72], &[1u8; 32]); + assert_eq!(&updates[72..104], &[2u8; 32]); + let decoded = realm_roots_from_state_updates(&updates).unwrap(); + assert_eq!(decoded.from_root, [1; 32]); + assert_eq!(decoded.to_root, [2; 32]); + } + + #[tokio::test] async fn transition_record_round_trip_and_read_bounds() { let dir = tempfile::tempdir().unwrap(); - let (proposal, body) = sample_object(1); - let store = ProposalStore::open(dir.path()).await.unwrap(); - store.save_proposal(&proposal, &body).await.unwrap(); - drop(store); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let loaded = store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap(); + let (proposal, body) = build_proposal_with_body(1); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + drop(proposal_backup); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let loaded = proposal_backup.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap(); assert_eq!(loaded.0.proposal_id, proposal.proposal_id); assert_eq!(loaded.1, body); let mut request = BodyChunkRequest { proposal_id: proposal.proposal_id, offset: 0, max_bytes: 64 }; - assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); - let path = store.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); + assert_eq!(proposal_backup.read_body_chunk(&request).await.unwrap().data, body[..64]); + let path = proposal_backup.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); let moved = dir.path().join("cached-object"); tokio::fs::rename(&path, &moved).await.unwrap(); request.offset = 64; - let second = store.read_body_chunk(&request).await.unwrap(); + let second = proposal_backup.read_body_chunk(&request).await.unwrap(); assert_eq!(second.data, body[64..128]); assert_eq!(second.body_hash, proposal.body_hash); assert_eq!(second.body_len, body.len() as u64); tokio::fs::rename(&moved, &path).await.unwrap(); request.offset = body.len() as u64; - let end = store.read_body_chunk(&request).await.unwrap(); + let end = proposal_backup.read_body_chunk(&request).await.unwrap(); assert!(end.eof); assert!(end.data.is_empty()); request.offset += 1; - assert!(store.read_body_chunk(&request).await.is_err()); + assert!(proposal_backup.read_body_chunk(&request).await.is_err()); request.offset = 0; request.max_bytes = 0; - assert!(store.read_body_chunk(&request).await.is_err()); + assert!(proposal_backup.read_body_chunk(&request).await.is_err()); request.max_bytes = BODY_CHUNK_MAX_BYTES + 1; - assert!(store.read_body_chunk(&request).await.is_err()); + assert!(proposal_backup.read_body_chunk(&request).await.is_err()); } #[tokio::test] async fn staging_order_and_slot_replacement() { let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (first, first_body) = sample_object(1); - let (second, second_body) = sample_object(2); - let first_stage = store.create_staged(&first, &first_body).await.unwrap(); - let second_stage = store.create_staged(&second, &second_body).await.unwrap(); - assert!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().is_none()); - assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); - assert_eq!(store.read_staged(&second_stage).await.unwrap().1, second_body); - store.install(second_stage).await.unwrap(); - assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![second.clone()]); - assert_eq!(store.read_staged(&first_stage).await.unwrap().1, first_body); - store.install(first_stage).await.unwrap(); - assert_eq!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().0.proposal_id, first.proposal_id); - assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![first.clone()]); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (first, first_body) = build_proposal_with_body(1); + let (second, second_body) = build_proposal_with_body(2); + let first_stage = proposal_backup.create_staged(&first, &first_body).await.unwrap(); + let second_stage = proposal_backup.create_staged(&second, &second_body).await.unwrap(); + assert!(proposal_backup.load_proposal(&[1; 32], &[2; 32]).await.unwrap().is_none()); + assert!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert_eq!(proposal_backup.read_staged(&second_stage).await.unwrap().1, second_body); + proposal_backup.install(second_stage).await.unwrap(); + assert_eq!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![second.clone()]); + assert_eq!(proposal_backup.read_staged(&first_stage).await.unwrap().1, first_body); + proposal_backup.install(first_stage).await.unwrap(); + assert_eq!(proposal_backup.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().0.proposal_id, first.proposal_id); + assert_eq!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![first.clone()]); let replaced = BodyChunkRequest { proposal_id: second.proposal_id, offset: 0, max_bytes: 64 }; - assert!(store.read_body_chunk(&replaced).await.is_err()); - store.save_proposal(&second, &second_body).await.unwrap(); - assert_eq!(store.read_body_chunk(&replaced).await.unwrap().data, second_body[..64]); - store.save_proposal(&first, &first_body).await.unwrap(); - assert_eq!(store.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().1, first_body); + assert!(proposal_backup.read_body_chunk(&replaced).await.is_err()); + proposal_backup.save_proposal(&second, &second_body).await.unwrap(); + assert_eq!(proposal_backup.read_body_chunk(&replaced).await.unwrap().data, second_body[..64]); + proposal_backup.save_proposal(&first, &first_body).await.unwrap(); + assert_eq!(proposal_backup.load_proposal(&[1; 32], &[2; 32]).await.unwrap().unwrap().1, first_body); } #[tokio::test] async fn damaged_transition_is_dropped_and_reinstallable() { let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (proposal, body) = sample_object(1); - store.save_proposal(&proposal, &body).await.unwrap(); - let path = store.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); + let proposal_backup = ProposalBackup::open(dir.path()).await.unwrap(); + let (proposal, body) = build_proposal_with_body(1); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + let path = proposal_backup.transition_path(&RealmTransition { from_root: [1; 32], to_root: [2; 32] }); let mut corrupted = encode_object(&proposal, &body); *corrupted.last_mut().unwrap() ^= 1; tokio::fs::write(&path, corrupted).await.unwrap(); let request = BodyChunkRequest { proposal_id: proposal.proposal_id, offset: 0, max_bytes: 64 }; // Cache hit: the installed header is served without touching disk. - assert_eq!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![proposal.clone()]); + assert_eq!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap(), vec![proposal.clone()]); // The verified load is the corruption discovery point: the damaged body - // is dropped (record + file) so the pair returns to absent. - assert!(store.load_proposal(&[1; 32], &[2; 32]).await.is_err()); + // is dropped (record + file) so the transition returns to absent. + assert!(proposal_backup.load_proposal(&[1; 32], &[2; 32]).await.is_err()); assert!(!path_exists(&path).await); - assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); - assert!(store.read_body_chunk(&request).await.is_err()); - let stage = store.create_staged(&proposal, &body).await.unwrap(); - assert_eq!(store.read_staged(&stage).await.unwrap().1, body); - store.install(stage).await.unwrap(); - assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); + assert!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert!(proposal_backup.read_body_chunk(&request).await.is_err()); + let stage = proposal_backup.create_staged(&proposal, &body).await.unwrap(); + assert_eq!(proposal_backup.read_staged(&stage).await.unwrap().1, body); + proposal_backup.install(stage).await.unwrap(); + assert_eq!(proposal_backup.read_body_chunk(&request).await.unwrap().data, body[..64]); tokio::fs::write(&path, &[]).await.unwrap(); - assert!(store.read_body_chunk(&request).await.is_err()); - assert!(store.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); + assert!(proposal_backup.read_body_chunk(&request).await.is_err()); + assert!(proposal_backup.lookup_transition(&[1; 32], &[2; 32]).await.unwrap().is_empty()); assert!(!path_exists(&path).await); - store.save_proposal(&proposal, &body).await.unwrap(); - assert_eq!(store.read_body_chunk(&request).await.unwrap().data, body[..64]); + proposal_backup.save_proposal(&proposal, &body).await.unwrap(); + assert_eq!(proposal_backup.read_body_chunk(&request).await.unwrap().data, body[..64]); } async fn path_exists(path: &str) -> bool { diff --git a/psy_node_common/src/realm/processor/recovery.rs b/psy_node_common/src/realm/processor/recovery.rs deleted file mode 100644 index 19247d5f..00000000 --- a/psy_node_common/src/realm/processor/recovery.rs +++ /dev/null @@ -1,2384 +0,0 @@ -//! Historical proposal verification and baseline FFS replay. - -use std::collections::{HashMap, HashSet}; - -use anyhow::Context; -use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; -use parth_common::memory_stores::traits::PsyMemoryMerkleStoreImm; -use parth_core::{ - crypto::hash::{ - merkle_proof::MerkleProofCore, - traits::{FieldQHasher, HashTo4Felts, MerkleHasher, MerkleZeroHasher, QFieldHashable}, - }, - data::hash::{ - fast_node_serializer::{ - QMerkleStoreFastDoubleNodeSerializer, QMerkleStoreFastSingleNodeSerializer, - QMerkleStoreFastZeroNodeSerializer, QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, - QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, - }, - merkle_node_key::SimpleMerkleNodeKey, - }, - felt::{FromPrimitiveValuesFelt, QFelt64, ToU64Value}, - protocol::core_types::{Q256BitHash, QFHashBase, QNetworkTypesConfig}, -}; -use psy_config::CHECKPOINTS_PER_EPOCH; -use psy_core::job::job_id::{ProvingJobCircuitType, QProvingJobDataID}; -use psy_data::{ - p2p::{ - sha256, BodyChunkRequest, BodyChunkResponse, NodeId, Proposal, ProposalLookupEntry, - ProposalLookupRequest, ProposalLookupResponse, ProposalLookupStatus, RealmTransition, - BODY_CHUNK_MAX_BYTES, MAX_PROPOSAL_BODY_BYTES, PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR, - PROPOSAL_LOOKUP_CONCURRENCY, PROPOSAL_LOOKUP_ROUND_SECS, PROPOSAL_LOOKUP_TIMEOUT_SECS, - PROPOSAL_LOOKUP_WINDOW_PAIRS, - }, - prepared_block::realm::{PsyPreparedRealmBlockStateUpdates, PsyRealmCoordinatorUpdate}, - v1::qdata::{ - contract::{ - deserialize_imt_leaf_ffs_entry_v2, IMTContractStateLeaf, IMT_LEAF_FFS_ENTRY_SIZE_V2, - }, - ffs_sizes::PSY_OBJECT_FFS_SIZE_USER_LEAF, - user::PQEDUserLeaf, - }, -}; -use psy_io::tokio::TokioLikeFileSystem; -use psy_node_core::{ - p2p::{traits::realm_coordinantor::RealmCoordinatorClient, validator_lookup::load_realm_validators_from_tree}, - psy_core_db::traits::full::{ - PsyNodeCoreRewardsTagTreeStoreReader, PsyNodeCoreRewardsTagTreeStoreWriter, PsyRealmProcessorStore, - }, - psy_temp_db::StandardProcessorTempDBStoreBase, - queue::{ephemeral::QStandardEphemeralQueueSubscriber, worker_queue::QStandardWorkerQueuePublisher}, - store::traits::proof_store::QParthProofStore, -}; -use psy_serialize::PsyCanonicalDatabaseSerializeBaseSingle; - -use crate::{ - backup::realm::load_realm_memory_trees_from_db, - realm::{ - network::{NetworkError, RealmNetworkCommands}, - processor::{ - consensus::{ - decode_proposal_state_updates, validator_tree_root_matches_proof_base, - verify_proposal_submission, require_declared_roots_match_zk_output, - }, - db::PsyRealmDatabaseProcessor, - proposal_store::{ProposalStore, StagedProposal}, - }, - }, -}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct CheckpointIdentity { - pub checkpoint_id: u64, - pub checkpoint_hash: [u8; 32], -} - -pub struct BaselineReplayRequest { - pub previous_checkpoint_id: u64, - pub updates: PsyPreparedRealmBlockStateUpdates, - pub reply: tokio::sync::oneshot::Sender>, -} - -pub struct VerifiedHistoryCandidate { - pub updates: PsyPreparedRealmBlockStateUpdates, - pub state_updates: Vec, - pub coordinator_update: PsyRealmCoordinatorUpdate, -} - -fn history_error(kind: &str, checkpoint_id: u64, detail: impl std::fmt::Display) -> anyhow::Error { - anyhow::anyhow!("{kind} at C={checkpoint_id}: {detail}") -} - -/// Why one history transition failed to verify: local material is absent or -/// stale (wait and retry), or the fetched candidate failed validation (prune -/// it and try the next candidate). -#[derive(Debug, thiserror::Error)] -pub(crate) enum RecoveryError { - #[error("{reason}")] - MissingLocalState { - #[source] - reason: anyhow::Error, - }, - #[error("{reason}")] - InvalidCandidate { - proposal_id: [u8; 32], - #[source] - reason: anyhow::Error, - }, -} - -pub(crate) fn invalid_candidate_id(error: &anyhow::Error) -> Option<[u8; 32]> { - match error.downcast_ref::()? { - RecoveryError::InvalidCandidate { proposal_id, .. } => Some(*proposal_id), - RecoveryError::MissingLocalState { .. } => None, - } -} - -fn missing_local_state(checkpoint_id: u64, detail: impl std::fmt::Display) -> RecoveryError { - RecoveryError::MissingLocalState { - reason: history_error("MissingHistoryProof", checkpoint_id, detail), - } -} - -fn gut_local_key( - key: SimpleMerkleNodeKey, - coordinator_height: u8, - realm_id: u64, -) -> anyhow::Result { - anyhow::ensure!( - key.level >= coordinator_height, - "InvalidStateUpdates: GUT node level {} is below coordinator height {}", - key.level, - coordinator_height - ); - let local_level = key.level - coordinator_height; - let expected_realm_id = if local_level >= 64 { - anyhow::ensure!(key.index == 0, "InvalidStateUpdates: GUT node index does not fit local level"); - 0 - } else { - key.index >> local_level - }; - anyhow::ensure!( - expected_realm_id == realm_id, - "InvalidStateUpdates: GUT node realm {expected_realm_id} does not match {realm_id}" - ); - let local_index = if local_level == 0 { - 0 - } else if local_level >= 64 { - key.index - } else { - key.index & ((1u64 << local_level) - 1) - }; - Ok(SimpleMerkleNodeKey { - level: local_level, - index: local_index, - }) -} - -fn require_width(bytes: &[u8], width: usize, what: &str) -> anyhow::Result<()> { - anyhow::ensure!( - bytes.is_empty() || bytes.len() % width == 0, - "InvalidStateUpdates: {what} length {} is not a multiple of {width}", - bytes.len() - ); - Ok(()) -} - -fn decode_double_id_node_ffs( - bytes: &[u8], - width: usize, - mut visit: impl FnMut(&[u8]), -) -> anyhow::Result<()> { - require_width(bytes, width, "tree node FFS")?; - for chunk in bytes.chunks_exact(width) { - visit(chunk); - } - Ok(()) -} - -fn seed_tree_from_merkle_proof( - tree: &mut SimpleMemoryMerkleRecorderStore, - proof: &MerkleProofCore, -) -> anyhow::Result<()> -where - H: MerkleZeroHasher + MerkleHasher, - Hash: Copy + PartialEq + Default + std::fmt::Debug, -{ - anyhow::ensure!( - proof.verify::(), - "MissingAuthenticatedState: previous merkle proof does not verify" - ); - let mut key = SimpleMerkleNodeKey::new(tree.get_height(), proof.index); - for sibling in &proof.siblings { - tree.set_node_value(key.sibling(), *sibling); - key = key.parent(); - } - tree.set_leaf(proof.index, proof.value); - Ok(()) -} - -fn require_declared_double_id_nodes_match( - tree: &SimpleMemoryMerkleRecorderStore, - nodes: &[(u8, u64, Hash)], -) -> anyhow::Result<()> -where - H: MerkleZeroHasher, - Hash: Copy + PartialEq + Default + std::fmt::Debug, -{ - for (level, index, value) in nodes { - let key = SimpleMerkleNodeKey { - level: *level, - index: *index, - }; - anyhow::ensure!( - tree.get_node_value(&key) == *value, - "InvalidStateUpdates: declared tree node {:?}={:?} does not match recomputed {:?}", - key, - value, - tree.get_node_value(&key) - ); - } - Ok(()) -} - -fn double_id_leaves_at_level( - nodes: &[(u8, u64, Hash)], - height: u8, -) -> HashMap { - let mut last_leaf = HashMap::new(); - for (level, index, value) in nodes { - if *level == height { - last_leaf.insert(*index, *value); - } - } - last_leaf -} - -pub fn replay_double_id_nodes_from_leaves( - tree: &mut SimpleMemoryMerkleRecorderStore, - nodes: &[(u8, u64, Hash)], -) -> anyhow::Result<()> -where - H: MerkleZeroHasher, - Hash: Copy + PartialEq + Default + std::fmt::Debug, -{ - let height = tree.get_height(); - for (index, value) in double_id_leaves_at_level(nodes, height) { - tree.set_leaf(index, value); - } - require_declared_double_id_nodes_match(tree, nodes) -} - -fn require_imt_leaf_ffs_consistency(bytes: &[u8]) -> anyhow::Result<()> -where - F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, - Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, - H: MerkleZeroHasher + FieldQHasher, -{ - require_width(bytes, IMT_LEAF_FFS_ENTRY_SIZE_V2, "IMT leaf FFS")?; - let mut first: HashMap<(u64, u64, u64), ([u8; 32], bool)> = HashMap::new(); - let mut first_key_index: HashMap<(u64, u64, [u8; 32]), u64> = HashMap::new(); - let mut first_new_keys: HashSet<(u64, u64, [u8; 32])> = HashSet::new(); - for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { - let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, leaf_value, next_key, next_index, is_new_key) = - deserialize_imt_leaf_ffs_entry_v2(chunk)?; - let leaf = IMTContractStateLeaf:: { - key: Hash::from_owned_32bytes(leaf_key), - value: Hash::from_owned_32bytes(leaf_value), - next_key: Hash::from_owned_32bytes(next_key), - next_index: F::from_u64_value(next_index), - }; - anyhow::ensure!( - leaf.qfhash::().into_owned_32bytes() == leaf_hash, - "InvalidStateUpdates: IMT leaf preimage does not bind leaf_hash" - ); - let id = (tree_id, tree_sub_id, leaf_index); - let key_id = (tree_id, tree_sub_id, leaf_key); - if let Some(first_index) = first_key_index.get(&key_id) { - if *first_index != leaf_index { - anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); - } - } else { - first_key_index.insert(key_id, leaf_index); - } - if let Some((first_key, first_new)) = first.get(&id) { - if is_new_key && !first_new_keys.contains(&key_id) { - anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); - } - if !*first_new && is_new_key { - anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); - } - if *first_key != leaf_key { - anyhow::bail!("InvalidStateUpdates: duplicate IMT key changed derived next fields"); - } - } else { - first.insert(id, (leaf_key, is_new_key)); - if is_new_key { - first_new_keys.insert(key_id); - } - } - } - let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], u64)> = HashMap::new(); - let mut seen_final = HashSet::new(); - for chunk in bytes.chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) { - let (tree_id, tree_sub_id, leaf_index, _, leaf_key, _, next_key, next_index, _) = - deserialize_imt_leaf_ffs_entry_v2(chunk)?; - if seen_final.insert((tree_id, tree_sub_id, leaf_index)) { - finals.insert((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)); - } - } - for ((tree_id, tree_sub_id, leaf_index), (leaf_key, next_key, next_index)) in &finals { - if *next_index == 0 { - anyhow::ensure!( - *next_key == [0u8; 32], - "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - continue; - } - let Some((successor_key, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) else { - continue; - }; - anyhow::ensure!( - successor_key == next_key, - "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index} next_index={next_index}" - ); - anyhow::ensure!( - successor_key != leaf_key || *next_index == *leaf_index, - "InvalidStateUpdates: IMT successor key collides with source user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - } - Ok(()) -} - -/// Changed contract-state leaves keyed by (user, contract, index). -pub fn contract_state_leaves_from_ffs( - updates: &PsyPreparedRealmBlockStateUpdates, -) -> anyhow::Result> -where - Hash: Copy + Q256BitHash, -{ - let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); - let mut contract_state_leaves: HashMap<(u64, u64, u64), Hash> = HashMap::new(); - if !updates.update_contract_state_tree_nodes_ffs.is_empty() { - let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); - decode_double_id_node_ffs( - &updates.update_contract_state_tree_nodes_ffs, - QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, - |chunk| { - let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( - chunk, - ); - grouped - .entry((node.key.tree_id, node.key.tree_sub_id)) - .or_default() - .push((node.key.level, node.key.index, node.value)); - }, - )?; - for ((user_id, contract_id), nodes) in grouped { - let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); - for (index, value) in double_id_leaves_at_level(&nodes, height) { - contract_state_leaves.insert((user_id, contract_id, index), value); - } - } - } - Ok(contract_state_leaves) -} - -pub fn require_state_update_record_coverage( - updates: &PsyPreparedRealmBlockStateUpdates, - checkpoint_id: u64, - imt_managed: &HashSet<(u64, u64, u64)>, -) -> anyhow::Result<()> -where - Hash: Copy + Q256BitHash, -{ - require_width( - &updates.update_global_user_tree_nodes_ffs, - QMS_FAST_SERIALIZER_ZERO_ID_NODE_SIZE, - "global user tree FFS", - )?; - require_width( - &updates.update_user_contract_tree_nodes_ffs, - QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, - "user contract tree FFS", - )?; - require_width( - &updates.update_contract_state_tree_nodes_ffs, - QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, - "contract state tree FFS", - )?; - require_width( - &updates.update_user_leaves_ffs, - PSY_OBJECT_FFS_SIZE_USER_LEAF, - "user leaf FFS", - )?; - require_width( - &updates.update_contract_state_imt_leaves_ffs, - IMT_LEAF_FFS_ENTRY_SIZE_V2, - "IMT leaf FFS", - )?; - if checkpoint_id == 0 { - return Ok(()); - } - let contract_state_leaves = contract_state_leaves_from_ffs(updates)?; - let mut imt_leaves: HashSet<(u64, u64, u64)> = HashSet::new(); - for chunk in updates - .update_contract_state_imt_leaves_ffs - .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) - { - let (tree_id, tree_sub_id, leaf_index, _, _, _, _, _, _) = - deserialize_imt_leaf_ffs_entry_v2(chunk)?; - imt_leaves.insert((tree_id, tree_sub_id, leaf_index)); - } - for (user_id, contract_id, index) in imt_managed { - anyhow::ensure!( - imt_leaves.contains(&(*user_id, *contract_id, *index)), - "InvalidStateUpdates: contract-state leaf user={user_id} contract={contract_id} index={index} has no IMT record" - ); - } - let mut user_contract_leaves: HashSet<(u64, u64)> = HashSet::new(); - if !updates.update_user_contract_tree_nodes_ffs.is_empty() { - let mut grouped: HashMap> = HashMap::new(); - decode_double_id_node_ffs( - &updates.update_user_contract_tree_nodes_ffs, - QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, - |chunk| { - let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( - chunk, - ); - grouped - .entry(node.key.tree_id) - .or_default() - .push((node.key.level, node.key.index)); - }, - )?; - for (user_id, nodes) in grouped { - let height = nodes.iter().map(|(level, _)| *level).max().unwrap_or(0); - for (level, index) in nodes { - if level == height { - user_contract_leaves.insert((user_id, index)); - } - } - } - } - let contract_pairs: HashSet<(u64, u64)> = contract_state_leaves - .keys() - .map(|(user_id, contract_id, _)| (*user_id, *contract_id)) - .collect(); - for (user_id, contract_id) in &user_contract_leaves { - anyhow::ensure!( - contract_pairs.contains(&(*user_id, *contract_id)), - "InvalidStateUpdates: user-contract leaf user={user_id} contract={contract_id} has no contract-state FFS" - ); - } - Ok(()) -} - -pub fn replay_state_updates_into_tree( - tree: &mut SimpleMemoryMerkleRecorderStore, - updates: &PsyPreparedRealmBlockStateUpdates, - coordinator_height: u8, - realm_user_tree_height: u8, - realm_id: u64, - checkpoint_id: u64, - imt_managed: &HashSet<(u64, u64, u64)>, -) -> anyhow::Result<()> -where - F: parth_core::felt::QFelt64 + FromPrimitiveValuesFelt, - Hash: Q256BitHash + QFHashBase + Copy + PartialEq + Default + std::fmt::Debug, - H: MerkleZeroHasher + FieldQHasher, -{ - require_state_update_record_coverage(updates, checkpoint_id, imt_managed)?; - anyhow::ensure!( - tree.get_root() == updates.old_realm_root, - "MissingAuthenticatedState: tree root {:?} is not old_realm_root {:?}", - tree.get_root(), - updates.old_realm_root - ); - - let gut_nodes = if updates.update_global_user_tree_nodes_ffs.is_empty() { - Vec::new() - } else { - QMerkleStoreFastZeroNodeSerializer::deserialize_zero_id_nodes_from_slice::( - &updates.update_global_user_tree_nodes_ffs, - ) - }; - let min_user_id = realm_id << realm_user_tree_height; - let mut last_user: HashMap> = HashMap::new(); - for bytes in updates - .update_user_leaves_ffs - .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) - { - let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; - last_user.insert(leaf.user_id.to_u64_value(), leaf); - } - let mut last_leaf: HashMap = HashMap::new(); - for node in &gut_nodes { - let local = gut_local_key(node.key, coordinator_height, realm_id)?; - if local.level == realm_user_tree_height { - last_leaf.insert(local.index, node.value); - } - } - for (index, value) in &last_leaf { - let previous_leaf = tree.get_leaf_value(*index); - if previous_leaf == *value { - continue; - } - let user_id = min_user_id + *index; - let leaf = last_user.get(&user_id).ok_or_else(|| { - anyhow::anyhow!("InvalidStateUpdates: GUT leaf {index} changed without preimage") - })?; - anyhow::ensure!( - leaf.qfhash::() == *value, - "InvalidStateUpdates: user {user_id} preimage does not bind the GUT leaf" - ); - } - for (index, value) in last_leaf { - tree.set_leaf(index, value); - } - - for node in &gut_nodes { - let local = gut_local_key(node.key, coordinator_height, realm_id)?; - anyhow::ensure!( - tree.get_node_value(&local) == node.value, - "InvalidStateUpdates: declared GUT node {:?}={:?} does not match recomputed {:?}", - local, - node.value, - tree.get_node_value(&local) - ); - } - - for (user_id, leaf) in &last_user { - anyhow::ensure!( - *user_id >= min_user_id, - "InvalidStateUpdates: user_id {user_id} is outside realm {realm_id}" - ); - let local_index = user_id - min_user_id; - let expected = leaf.qfhash::(); - anyhow::ensure!( - tree.get_leaf_value(local_index) == expected, - "InvalidStateUpdates: user {user_id} preimage does not bind the recomputed GUT leaf" - ); - } - - require_imt_leaf_ffs_consistency::(&updates.update_contract_state_imt_leaves_ffs)?; - - anyhow::ensure!( - tree.get_root() == updates.new_realm_root, - "InvalidStateUpdates: recomputed root {:?} is not new_realm_root {:?}", - tree.get_root(), - updates.new_realm_root - ); - Ok(()) -} - -/// Non-empty changed leaves of every contract-state tree the previous -/// checkpoint's IMT index manages: those must ship IMT records — including -/// first-time inserts, whose `is_new_key` write is mandatory — or the IMT -/// index keeps proving a stale value or misses the key entirely. Trees the -/// IMT index has no entries for are positional and fully exempt; a leaf -/// cleared to zero keeps the no-IMT-required behavior. The next-append -/// pointer is read before this block's IMT FFS applies, so it still reflects -/// the state the update builds on. -pub async fn imt_managed_leaves_from_db( - db: &S, - updates: &PsyPreparedRealmBlockStateUpdates, -) -> anyhow::Result> -where - S: psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader + Sync, - F: parth_core::felt::QFelt64, - Hash: Q256BitHash + Copy + PartialEq, -{ - let empty_leaf = Hash::from_owned_32bytes([0u8; 32]); - let mut changed_trees: HashMap<(u64, u64), Vec> = HashMap::new(); - for ((user_id, contract_id, index), new_value) in contract_state_leaves_from_ffs(updates)? { - if new_value == empty_leaf { - continue; - } - changed_trees - .entry((user_id, contract_id)) - .or_default() - .push(index); - } - let mut managed = HashSet::new(); - for ((user_id, contract_id), leaves) in changed_trees { - let next_append_index = db - .contract_state_imt_get_next_append_index(user_id, contract_id) - .await - .with_context(|| format!("previous-checkpoint IMT append index read failed user={user_id} contract={contract_id}"))?; - if next_append_index == 0 { - continue; - } - for index in leaves { - managed.insert((user_id, contract_id, index)); - } - } - Ok(managed) -} - -async fn load_previous_contract_heights( - db: &S, - previous_checkpoint_id: u64, - contract_ids: impl IntoIterator, -) -> anyhow::Result> -where - S: psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader + Sync, - F: Send + Sync, - Hash: Send + Sync, -{ - let mut unique = Vec::new(); - let mut seen = HashSet::new(); - for contract_id in contract_ids { - if seen.insert(contract_id) { - unique.push(contract_id); - } - } - if unique.is_empty() { - return Ok(HashMap::new()); - } - let fetched = db - .get_contract_tree_heights(previous_checkpoint_id, &unique) - .await - .map_err(|error| { - anyhow::anyhow!( - "MissingAuthenticatedState: contract heights unavailable at previous checkpoint {previous_checkpoint_id}: {error}" - ) - })?; - let mut heights = HashMap::with_capacity(unique.len()); - for (i, contract_id) in unique.into_iter().enumerate() { - heights.insert(contract_id, fetched.get(i).copied().unwrap_or(0)); - } - Ok(heights) -} - -fn require_previous_contract_height(heights: &HashMap, contract_id: u64) -> anyhow::Result { - let height = heights.get(&contract_id).copied().unwrap_or(0); - anyhow::ensure!( - height > 0, - "MissingAuthenticatedState: contract {contract_id} height is zero" - ); - Ok(height) -} - - - - - -impl< - N: QNetworkTypesConfig, - S: PsyRealmProcessorStore + Send + Sync, - STagTreeRewards: PsyNodeCoreRewardsTagTreeStoreWriter - + PsyNodeCoreRewardsTagTreeStoreReader - + Send - + Sync, - GUTAUpdateQueue: QStandardEphemeralQueueSubscriber + Send + Sync, - ProofWorkQueue: QStandardWorkerQueuePublisher + Send + Sync, - TempDatabase: StandardProcessorTempDBStoreBase, - ProofStore: QParthProofStore, - FileSystem: TokioLikeFileSystem + Send + Sync + 'static, - CoordinatorClient: RealmCoordinatorClient + Send + Sync, - > - PsyRealmDatabaseProcessor< - N, - S, - STagTreeRewards, - GUTAUpdateQueue, - ProofWorkQueue, - TempDatabase, - ProofStore, - FileSystem, - CoordinatorClient, - > -where - N::HasherBase: 'static + Send + Sync + MerkleZeroHasher + FieldQHasher, -{ - pub async fn verify_state_updates_from_baseline( - &self, - previous_checkpoint_id: u64, - updates: &PsyPreparedRealmBlockStateUpdates, - ) -> anyhow::Result<()> { - let imt_managed = crate::realm::processor::recovery::imt_managed_leaves_from_db::(self.db.as_ref(), updates) - .await?; - let mut trees = load_realm_memory_trees_from_db::( - &*self.db, - previous_checkpoint_id, - self.state.realm_id_u64, - ) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState at previous checkpoint {previous_checkpoint_id}: {error}"))?; - let mut tree = trees.into_tuple().0; - let checkpoint_id = previous_checkpoint_id.saturating_add(1); - replay_state_updates_into_tree::( - &mut tree, - updates, - N::COORDINATOR_GLOBAL_USER_TREE_HEIGHT, - N::REALM_GLOBAL_USER_TREE_HEIGHT, - self.state.realm_id_u64, - checkpoint_id, - &imt_managed, - ) - .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}"))?; - self.verify_double_id_trees_from_previous(previous_checkpoint_id, updates) - .await - .with_context(|| format!("baseline replay failed at previous checkpoint {previous_checkpoint_id}")) - } - - async fn verify_double_id_trees_from_previous( - &self, - previous_checkpoint_id: u64, - updates: &PsyPreparedRealmBlockStateUpdates, - ) -> anyhow::Result<()> { - let mut last_user: HashMap> = HashMap::new(); - for bytes in updates - .update_user_leaves_ffs - .chunks_exact(PSY_OBJECT_FFS_SIZE_USER_LEAF) - { - let leaf = PQEDUserLeaf::::psy_ser_from_slice(bytes)?; - last_user.insert(leaf.user_id.to_u64_value(), leaf); - } - let mut user_contract_leaves: HashMap<(u64, u64), N::QHash> = HashMap::new(); - if !updates.update_user_contract_tree_nodes_ffs.is_empty() { - let mut grouped: HashMap> = HashMap::new(); - decode_double_id_node_ffs( - &updates.update_user_contract_tree_nodes_ffs, - QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, - |chunk| { - let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( - chunk, - ); - grouped - .entry(node.key.tree_id) - .or_default() - .push((node.key.level, node.key.index, node.value)); - }, - )?; - for (user_id, nodes) in grouped { - anyhow::ensure!( - nodes.iter().all(|(level, _, _)| *level <= N::GLOBAL_CONTRACT_TREE_HEIGHT), - "InvalidStateUpdates: user-contract node level exceeds tree height" - ); - let mut tree = SimpleMemoryMerkleRecorderStore::::new( - N::GLOBAL_CONTRACT_TREE_HEIGHT, - ); - let leaves = double_id_leaves_at_level(&nodes, N::GLOBAL_CONTRACT_TREE_HEIGHT); - if leaves.is_empty() && !nodes.is_empty() { - anyhow::bail!( - "MissingAuthenticatedState: user-contract internals for user {user_id} have no leaf preimages" - ); - } - for index in leaves.keys() { - let proof = self - .db - .user_contract_tree_get_merkle_proof(previous_checkpoint_id, user_id, *index) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract proof user={user_id} index={index}: {error}"))?; - seed_tree_from_merkle_proof(&mut tree, &proof)?; - } - replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; - for (index, value) in leaves { - user_contract_leaves.insert((user_id, index), value); - } - let new_root = tree.get_root(); - let bound = if let Some(leaf) = last_user.get(&user_id) { - leaf.user_state_tree_root - } else { - self.db - .get_user_leaf(previous_checkpoint_id, user_id) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user {user_id} leaf unavailable: {error}"))? - .user_state_tree_root - }; - anyhow::ensure!( - new_root == bound, - "InvalidStateUpdates: user {user_id} contract tree root does not bind the user leaf" - ); - } - } - let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, N::QHash)>> = HashMap::new(); - if !updates.update_contract_state_tree_nodes_ffs.is_empty() { - decode_double_id_node_ffs( - &updates.update_contract_state_tree_nodes_ffs, - QMS_FAST_SERIALIZER_DOUBLE_ID_NODE_SIZE, - |chunk| { - let node = QMerkleStoreFastDoubleNodeSerializer::deserialize_double_id_node_from_slice::( - chunk, - ); - grouped - .entry((node.key.tree_id, node.key.tree_sub_id)) - .or_default() - .push((node.key.level, node.key.index, node.value)); - }, - )?; - } - let mut finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)> = HashMap::new(); - if !updates.update_contract_state_imt_leaves_ffs.is_empty() { - for chunk in updates - .update_contract_state_imt_leaves_ffs - .chunks_exact(IMT_LEAF_FFS_ENTRY_SIZE_V2) - { - let (tree_id, tree_sub_id, leaf_index, leaf_hash, leaf_key, _, next_key, next_index, is_new_key) = - deserialize_imt_leaf_ffs_entry_v2(chunk)?; - finals.entry((tree_id, tree_sub_id, leaf_index)).or_insert(( - leaf_hash, - leaf_key, - next_key, - next_index, - is_new_key, - )); - } - } - let heights = load_previous_contract_heights( - self.db.as_ref(), - previous_checkpoint_id, - grouped - .keys() - .map(|(_, contract_id)| *contract_id) - .chain(finals.keys().map(|(_, contract_id, _)| *contract_id)), - ) - .await?; - let mut contract_state_leaves: HashMap<(u64, u64, u64), N::QHash> = HashMap::new(); - for ((user_id, contract_id), nodes) in grouped { - let height = require_previous_contract_height(&heights, contract_id)?; - anyhow::ensure!( - nodes.iter().all(|(level, _, _)| *level <= height), - "InvalidStateUpdates: contract-state node level exceeds authenticated height {height}" - ); - let mut tree = SimpleMemoryMerkleRecorderStore::::new(height); - let leaves = double_id_leaves_at_level(&nodes, height); - if leaves.is_empty() && !nodes.is_empty() { - anyhow::bail!( - "MissingAuthenticatedState: contract-state internals for user={user_id} contract={contract_id} have no leaf preimages" - ); - } - for index in leaves.keys() { - let proof = self - .db - .contract_state_tree_get_merkle_proof( - previous_checkpoint_id, - user_id, - contract_id, - height, - *index, - ) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: contract-state proof user={user_id} contract={contract_id} index={index}: {error}"))?; - seed_tree_from_merkle_proof(&mut tree, &proof)?; - } - replay_double_id_nodes_from_leaves(&mut tree, &nodes)?; - for (index, value) in leaves { - contract_state_leaves.insert((user_id, contract_id, index), value); - } - let new_root = tree.get_root(); - let bound = if let Some(leaf) = user_contract_leaves.get(&(user_id, contract_id)) { - *leaf - } else { - self.db - .user_contract_tree_get_leaf_hash(previous_checkpoint_id, user_id, contract_id) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: user-contract leaf user={user_id} contract={contract_id}: {error}"))? - }; - anyhow::ensure!( - new_root == bound, - "InvalidStateUpdates: contract-state root does not bind user {user_id} contract {contract_id}" - ); - } - self.verify_imt_from_previous(previous_checkpoint_id, finals, &contract_state_leaves, &heights) - .await - } - - async fn verify_imt_from_previous( - &self, - previous_checkpoint_id: u64, - finals: HashMap<(u64, u64, u64), ([u8; 32], [u8; 32], [u8; 32], u64, bool)>, - contract_state_leaves: &HashMap<(u64, u64, u64), N::QHash>, - heights: &HashMap, - ) -> anyhow::Result<()> { - for ((tree_id, tree_sub_id, leaf_index), (leaf_hash, leaf_key, next_key, next_index, is_new_key)) in &finals { - let height = require_previous_contract_height(heights, *tree_sub_id)?; - let expected = if let Some(value) = contract_state_leaves.get(&(*tree_id, *tree_sub_id, *leaf_index)) { - *value - } else { - self.db - .contract_state_tree_get_leaf_hash( - previous_checkpoint_id, - *tree_id, - *tree_sub_id, - height, - *leaf_index, - ) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))? - }; - anyhow::ensure!( - expected.into_owned_32bytes() == *leaf_hash, - "InvalidStateUpdates: IMT leaf_hash does not bind contract-state leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - let key = N::QHash::from_owned_32bytes(*leaf_key); - let old_index = self - .db - .contract_state_imt_get_leaf_index_for_key(previous_checkpoint_id, *tree_id, *tree_sub_id, &key) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT key index user={tree_id} contract={tree_sub_id}: {error}"))?; - if *is_new_key { - anyhow::ensure!( - old_index.is_none(), - "InvalidStateUpdates: IMT is_new_key already indexed user={tree_id} contract={tree_sub_id}" - ); - let previous_at_index = self - .db - .contract_state_imt_get_leaf_preimage( - previous_checkpoint_id, - *tree_id, - *tree_sub_id, - *leaf_index, - ) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT previous leaf user={tree_id} contract={tree_sub_id} index={leaf_index}: {error}"))?; - anyhow::ensure!( - previous_at_index.is_none(), - "InvalidStateUpdates: IMT new key overwrites an authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - } else { - let old_index = old_index.ok_or_else(|| { - anyhow::anyhow!( - "InvalidStateUpdates: IMT key is not new but has no authenticated index user={tree_id} contract={tree_sub_id} index={leaf_index}" - ) - })?; - anyhow::ensure!( - old_index == *leaf_index, - "InvalidStateUpdates: IMT key index moved user={tree_id} contract={tree_sub_id}" - ); - } - if *next_index == 0 { - anyhow::ensure!( - *next_key == [0u8; 32], - "InvalidStateUpdates: IMT terminal next_key must be zero user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - continue; - } - if let Some((_, successor_key, _, _, _)) = finals.get(&(*tree_id, *tree_sub_id, *next_index)) { - anyhow::ensure!( - successor_key == next_key, - "InvalidStateUpdates: IMT next_key does not match successor leaf user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - continue; - } - let successor = self - .db - .contract_state_imt_get_leaf_preimage( - previous_checkpoint_id, - *tree_id, - *tree_sub_id, - *next_index, - ) - .await - .map_err(|error| anyhow::anyhow!("MissingAuthenticatedState: IMT successor user={tree_id} contract={tree_sub_id} next_index={next_index}: {error}"))?; - let successor = successor.ok_or_else(|| { - anyhow::anyhow!("MissingAuthenticatedState: IMT successor missing user={tree_id} contract={tree_sub_id} next_index={next_index}") - })?; - anyhow::ensure!( - successor.key.into_owned_32bytes() == *next_key, - "InvalidStateUpdates: IMT next_key does not match authenticated successor user={tree_id} contract={tree_sub_id} index={leaf_index}" - ); - } - Ok(()) - } - - pub async fn verify_history_proposal( - &self, - included: &CheckpointIdentity, - proposal: &Proposal, - body: &[u8], - ) -> anyhow::Result<( - PsyPreparedRealmBlockStateUpdates, - Vec, - PsyRealmCoordinatorUpdate, - )> { - anyhow::ensure!( - proposal.realm_id == self.state.realm_id_u64 as u32, - "InvalidStateUpdates at C={}: proposal realm mismatch", - included.checkpoint_id - ); - anyhow::ensure!( - proposal.chain_id == self.state.chain_id, - "InvalidStateUpdates at C={}: proposal chain mismatch", - included.checkpoint_id - ); - let coordinator_update = self - .coordinator_client - .rc_get_realm_sync_info(included.checkpoint_id, self.state.realm_id_u64) - .await - .map_err(|error| missing_local_state( - included.checkpoint_id, - format!("Coordinator C materials unavailable: {error:#}"), - ))?; - anyhow::ensure!( - coordinator_update.checkpoint_sync_info.checkpoint_id == included.checkpoint_id, - "MissingHistoryProof at C={}: coordinator checkpoint id mismatch", - included.checkpoint_id - ); - anyhow::ensure!( - coordinator_update - .checkpoint_sync_info - .checkpoint_leaf_hash - .into_owned_32bytes() - == included.checkpoint_hash, - "MissingHistoryProof at C={}: coordinator leaf hash does not match included.checkpoint_hash", - included.checkpoint_id - ); - let authenticated_leaf = self - .checkpoint_tree_backup_manager - .checkpoint_tree - .get_leaf(included.checkpoint_id); - if authenticated_leaf.value.into_owned_32bytes() != included.checkpoint_hash { - return Err(missing_local_state( - included.checkpoint_id, - "included.checkpoint_hash does not match the authenticated checkpoint tree leaf", - ) - .into()); - } - let roots = self - .db - .get_checkpoint_global_state_roots(proposal.base_checkpoint_id) - .await - .map_err(|error| missing_local_state( - included.checkpoint_id, - format!("proof-base P={} roots unavailable: {error:#}", proposal.base_checkpoint_id), - ))?; - anyhow::ensure!( - validator_tree_root_matches_proof_base( - &proposal.validator_tree_root, - &roots.validator_tree_root.into_owned_32bytes(), - ), - "MissingHistoryProof at C={}: proposal.validator_tree_root does not match P={}", - included.checkpoint_id, - proposal.base_checkpoint_id - ); - let (_, _, user_ids, _) = - load_realm_validators_from_tree::( - &*self.db, - self.state.chain_id, - proposal.base_checkpoint_id, - proposal.realm_id, - &roots.validator_tree_root, - ) - .await - .map_err(|error| missing_local_state( - included.checkpoint_id, - format!("proof-base P={} validator tree unavailable: {error:#}", proposal.base_checkpoint_id), - ))?; - let proposer_user_id = user_ids - .iter() - .find(|(sub_id, _)| *sub_id == proposal.proposer_sub_id) - .map(|(_, user_id)| *user_id) - .ok_or_else(|| { - history_error( - "MissingHistoryProof", - included.checkpoint_id, - format!("proposer sub_id {} is not a validator", proposal.proposer_sub_id), - ) - })?; - let decoded = - verify_proposal_submission::(proposal, body, proposer_user_id, self.proof_verifier.as_ref())?; - let updates = decode_proposal_state_updates::(&decoded.state_updates)?; - anyhow::ensure!( - updates.realm_id == proposal.realm_id as u64, - "InvalidStateUpdates at C={}: FFS realm_id {} does not match proposal {}", - included.checkpoint_id, - updates.realm_id, - proposal.realm_id - ); - anyhow::ensure!( - updates.realm_sub_id == proposal.proposer_sub_id as u64, - "InvalidStateUpdates at C={}: FFS realm_sub_id {} does not match proposer {}", - included.checkpoint_id, - updates.realm_sub_id, - proposal.proposer_sub_id - ); - let output = psy_data::guta::realm_finalize::protocol_decode_finalize_output::( - &decoded.output, - )?; - require_declared_roots_match_zk_output(&updates, &output)?; - anyhow::ensure!( - updates.new_realm_root.into_owned_32bytes() != [0u8; 32] - || updates.old_realm_root == updates.new_realm_root, - "InvalidStateUpdates at C={}: empty new root", - included.checkpoint_id - ); - let realm_proof = &coordinator_update.merkle_proof_to_realm_root; - anyhow::ensure!( - realm_proof.verify::(), - "MissingHistoryProof at C={}: realm-root path does not verify", - included.checkpoint_id - ); - anyhow::ensure!( - realm_proof.index == self.state.realm_id_u64, - "MissingHistoryProof at C={}: realm-root path index mismatch", - included.checkpoint_id - ); - anyhow::ensure!( - realm_proof.value == updates.new_realm_root, - "MissingHistoryProof at C={}: authenticated realm root does not match proposal new_realm_root", - included.checkpoint_id - ); - anyhow::ensure!( - realm_proof.root == coordinator_update.checkpoint_sync_info.state_roots.user_tree_root, - "MissingHistoryProof at C={}: realm-root path is not bound to C user_tree_root", - included.checkpoint_id - ); - let previous = included - .checkpoint_id - .checked_sub(1) - .ok_or_else(|| history_error("MissingHistoryProof", included.checkpoint_id, "C=0 has no predecessor"))?; - self.verify_state_updates_from_baseline(previous, &updates) - .await - .with_context(|| { - format!( - "InvalidStateUpdates at C={} proposal_id={}", - included.checkpoint_id, - hex::encode(proposal.proposal_id) - ) - })?; - Ok((updates, decoded.state_updates, coordinator_update)) - } - - pub async fn ensure_uncommitted_processing_ids(&mut self, checkpoint_id: u64) -> anyhow::Result<()> { - let pending_id = self.state.processing_unique_pending_id; - let mapped_checkpoint = self.db.get_checkpoint_id_for_unique_pending_id(pending_id).await?; - if pending_id != 0 && mapped_checkpoint == Some(checkpoint_id) { - return Ok(()); - } - let (pending_id, proc_checkpoint_unique_id) = - if let Some(ids) = self.db.get_unique_pending_id_for_checkpoint_id(checkpoint_id).await? { - ids - } else if pending_id != 0 && mapped_checkpoint.is_none() { - return Ok(()); - } else { - self.db.inc_unique_pending_id(1).await? - }; - self.state.processing_unique_pending_id = pending_id; - self.state.processing_proc_checkpoint_unique_id = proc_checkpoint_unique_id; - self.temp_db - .set_unique_pending_ids(&self.state.realm_identifier, pending_id, proc_checkpoint_unique_id) - .await?; - Ok(()) - } - - pub async fn apply_history_proposal( - &mut self, - included: &CheckpointIdentity, - verified: VerifiedHistoryCandidate, - ) -> anyhow::Result<(PsyPreparedRealmBlockStateUpdates, Vec)> { - let VerifiedHistoryCandidate { - updates, - state_updates, - coordinator_update, - } = verified; - self.ensure_uncommitted_processing_ids(included.checkpoint_id).await?; - self.state.processing_checkpoint_id = included.checkpoint_id; - self.state.processing_checkpoint_root = coordinator_update.checkpoint_sync_info.checkpoint_tree_root; - self.state.processing_realm_start_root = updates.old_realm_root; - self.state.processing_realm_end_root = updates.new_realm_root; - if self.state.last_committed_checkpoint_id >= included.checkpoint_id { - anyhow::ensure!( - self.state.last_committed_realm_end_root == updates.new_realm_root, - "InvalidStateUpdates at C={}: committed realm root does not match candidate; refusing second FFS", - included.checkpoint_id - ); - } else { - self.commit_state( - &coordinator_update, - &updates, - ProvingJobCircuitType::GUTANoChange, - vec![], - ) - .await?; - } - Ok((updates, state_updates)) - } - - /// Verify the candidate for one root pair, reading it from the staged bytes - /// when the fetch stage supplied them and from the pair slot otherwise. - pub async fn verify_history_transition( - &self, - included: &CheckpointIdentity, - pair: RealmTransition, - staged: Option<&StagedProposal>, - store: &ProposalStore, - ) -> anyhow::Result>> { - let loaded = match staged { - Some(staged) => Some(store.read_staged(staged).await?), - None => store - .load_proposal(&pair.old_root, &pair.new_root) - .await?, - }; - let Some((proposal, body)) = loaded else { - return Ok(None); - }; - match self - .verify_history_proposal(included, &proposal, &body) - .await - { - Ok((updates, state_updates, coordinator_update)) => { - Ok(Some(VerifiedHistoryCandidate { - updates, - state_updates, - coordinator_update, - })) - } - Err(error) => { - if error - .downcast_ref::() - .is_some_and(|classified| matches!(classified, RecoveryError::MissingLocalState { .. })) - { - return Err(error); - } - tracing::warn!( - "history verify rejected C={} pair=({},{}) proposal={} error={error}", - included.checkpoint_id, - hex::encode(pair.old_root), - hex::encode(pair.new_root), - hex::encode(proposal.proposal_id) - ); - Err(anyhow::Error::new(RecoveryError::InvalidCandidate { - proposal_id: proposal.proposal_id, - reason: anyhow::anyhow!("history verify rejected C={}: {error:#}", included.checkpoint_id), - })) - } - } - } -} - -/// Attempts per checkpoint: the staged window candidate plus re-fetches that -/// exclude the candidates already rejected for that pair. -#[cfg(test)] -mod tests { - use super::*; - use crate::realm::processor::catchup::*; - use parth_common::memory_stores::mem_tree_recorder::SimpleMemoryMerkleRecorderStore; - use parth_core::{ - crypto::hash::traits::{MerkleZeroHasher, QFieldHashable}, - data::hash::merkle_node_key::SimpleMerkleNode, - pgoldilocks::{PGoldilocksFelt, PGoldilocksHash, PoseidonHasher}, - protocol::core_types::Q256BitHash, - }; - use psy_data::prepared_block::realm::PsyPreparedRealmBlockStateUpdates; - - fn verify_double_id_node_ffs_bytes( - bytes: &[u8], - width: usize, - parse: impl Fn(&[u8]) -> (u64, u64, u8, u64, Hash), - ) -> anyhow::Result<()> - where - H: MerkleZeroHasher, - Hash: Q256BitHash + Copy + PartialEq + Default + std::fmt::Debug, - { - require_width(bytes, width, "tree node FFS")?; - let mut grouped: HashMap<(u64, u64), Vec<(u8, u64, Hash)>> = HashMap::new(); - for chunk in bytes.chunks_exact(width) { - let (tree_id, tree_sub_id, level, index, value) = parse(chunk); - grouped.entry((tree_id, tree_sub_id)).or_default().push((level, index, value)); - } - for nodes in grouped.values() { - let height = nodes.iter().map(|(level, _, _)| *level).max().unwrap_or(0); - let mut tree = SimpleMemoryMerkleRecorderStore::::new(height.max(1)); - replay_double_id_nodes_from_leaves(&mut tree, nodes)?; - } - Ok(()) - } - - fn empty_updates(old: PGoldilocksHash, new: PGoldilocksHash) -> PsyPreparedRealmBlockStateUpdates { - PsyPreparedRealmBlockStateUpdates { - realm_id: 0, - realm_sub_id: 0, - unique_pending_id: 0, - proc_checkpoint_unique_id: Default::default(), - old_realm_root: old, - new_realm_root: new, - update_global_user_tree_nodes_ffs: vec![], - update_user_contract_tree_nodes_ffs: vec![], - update_contract_state_tree_nodes_ffs: vec![], - update_user_leaves_ffs: vec![], - update_contract_state_imt_leaves_ffs: vec![], - } - } - - #[test] - fn history_bad_ffs() { - let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); - let old = tree.get_root(); - let mut updates = empty_updates(old, old); - let poison = SimpleMerkleNode { - key: SimpleMerkleNodeKey { level: 8, index: 0 }, - value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), - }; - updates.update_global_user_tree_nodes_ffs = - QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&poison); - let error = replay_state_updates_into_tree::( - &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), - ) - .expect_err("poisoned GUT node must fail baseline replay"); - assert!( - error.to_string().contains("InvalidStateUpdates"), - "{error}" - ); - let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { - tree_id: 1, - level: 8, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), - }; - let poison_contract = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { - tree_id: 1, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x44; 32]), - }; - let mut contract_ffs = - QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&contract_leaf); - contract_ffs.extend_from_slice( - &QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&poison_contract), - ); - let leaf_poison = SimpleMerkleNode { - key: SimpleMerkleNodeKey { level: 16, index: 0 }, - value: PGoldilocksHash::from_owned_32bytes([0x22; 32]), - }; - let mut leaf_updates = empty_updates(old, old); - leaf_updates.update_global_user_tree_nodes_ffs = - QMerkleStoreFastZeroNodeSerializer::serialize_zero_id_node_to_vec(&leaf_poison); - let preimage_error = replay_state_updates_into_tree::( - &mut SimpleMemoryMerkleRecorderStore::::new(8), - &leaf_updates, - 8, - 8, - 0, - 1, - &HashSet::new(), - ) - .expect_err("changed GUT leaf without preimage must fail"); - assert!( - preimage_error.to_string().contains("InvalidStateUpdates"), - "{preimage_error}" - ); - let contract_error = verify_double_id_node_ffs_bytes::( - &contract_ffs, - QMS_FAST_SERIALIZER_SINGLE_ID_NODE_SIZE, - |chunk| { - let node = QMerkleStoreFastSingleNodeSerializer::deserialize_single_id_node_from_slice::( - chunk, - ); - (node.key.tree_id, 0, node.key.level, node.key.index, node.value) - }, - ) - .expect_err("poisoned contract node must fail baseline replay"); - assert!( - contract_error.to_string().contains("InvalidStateUpdates"), - "{contract_error}" - ); - } - - #[test] - fn history_duplicate_imt() { - let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); - let old = tree.get_root(); - let mut updates = empty_updates(old, old); - let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); - let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); - let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); - let first_leaf = IMTContractStateLeaf:: { - key: first_key, - value: first_value, - next_key: first_next_key, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), - }; - let first_hash = first_leaf.qfhash::(); - let first_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 0, false, - ); - let second_entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 3, &first_hash, &first_key, &first_value, &first_next_key, 7, true, - ); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&first_entry); - bytes.extend_from_slice(&second_entry); - updates.update_contract_state_imt_leaves_ffs = bytes; - let error = replay_state_updates_into_tree::( - &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), - ) - .expect_err("conflicting IMT history must fail"); - assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); - let moved = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 9, &first_hash, &first_key, &first_value, &first_next_key, 0, false, - ); - let mut moved_bytes = Vec::new(); - moved_bytes.extend_from_slice(&first_entry); - moved_bytes.extend_from_slice(&moved); - let mut moved_updates = empty_updates(old, old); - moved_updates.update_contract_state_imt_leaves_ffs = moved_bytes; - let moved_error = replay_state_updates_into_tree::( - &mut SimpleMemoryMerkleRecorderStore::::new(8), - &moved_updates, - 8, - 8, - 0, - 1, - &HashSet::new(), - ) - .expect_err("same IMT key at a second leaf index must fail"); - assert!(moved_error.to_string().contains("InvalidStateUpdates"), "{moved_error}"); - let successor_key = PGoldilocksHash::from_owned_32bytes([0x44u8; 32]); - let successor_leaf = IMTContractStateLeaf:: { - key: successor_key, - value: first_value, - next_key: first_next_key, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), - }; - let successor_hash = successor_leaf.qfhash::(); - let successor = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 4, &successor_hash, &successor_key, &first_value, &first_next_key, 0, false, - ); - let mismatched_next = PGoldilocksHash::from_owned_32bytes([0x99u8; 32]); - let mismatched_leaf = IMTContractStateLeaf:: { - key: first_key, - value: first_value, - next_key: mismatched_next, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(4), - }; - let mismatched_hash = mismatched_leaf.qfhash::(); - let mismatched = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 3, &mismatched_hash, &first_key, &first_value, &mismatched_next, 4, false, - ); - let mut successor_bytes = Vec::new(); - successor_bytes.extend_from_slice(&mismatched); - successor_bytes.extend_from_slice(&successor); - let successor_error = require_imt_leaf_ffs_consistency::(&successor_bytes) - .expect_err("IMT next_key must match the successor leaf in the same FFS"); - assert!(successor_error.to_string().contains("InvalidStateUpdates"), "{successor_error}"); - } - - #[test] - fn history_ffs_coverage_requires_imt_for_changed_contract_leaf() { - let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); - let mut updates = empty_updates(old, old); - let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 1, - tree_sub_id: 2, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), - }; - updates.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); - let error = require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) - .expect_err("changed contract-state leaf without IMT must fail"); - assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); - assert!(error.to_string().contains("no IMT record"), "{error}"); - let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); - let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); - let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); - let first_leaf = IMTContractStateLeaf:: { - key: first_key, - value: first_value, - next_key: first_next_key, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), - }; - let first_hash = first_leaf.qfhash::(); - updates.update_contract_state_imt_leaves_ffs = - psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, - ) - .to_vec(); - require_state_update_record_coverage(&updates, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) - .expect("matching IMT record must close the coverage set"); - require_state_update_record_coverage(&updates, 0, &HashSet::new()) - .expect("genesis may carry contract-state FFS with empty IMT"); - let mut genesis = empty_updates(old, old); - genesis.update_contract_state_tree_nodes_ffs = updates.update_contract_state_tree_nodes_ffs.clone(); - require_state_update_record_coverage(&genesis, 0, &HashSet::new()) - .expect("genesis contract-state leaves with empty IMT are a legal empty IMT"); - let genesis_error = require_state_update_record_coverage(&genesis, 1, &HashSet::from_iter([(1u64, 2u64, 0u64)])) - .expect_err("the same missing IMT must fail after genesis"); - assert!(genesis_error.to_string().contains("no IMT record"), "{genesis_error}"); - require_state_update_record_coverage(&empty_updates(old, old), 1, &HashSet::new()) - .expect("empty contract-state FFS and empty IMT is a legal no-op"); - } - - #[test] - fn history_ffs_user_contract_requires_contract_state() { - let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); - let mut updates = empty_updates(old, old); - let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { - tree_id: 1, - level: 8, - index: 2, - }, - value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), - }; - updates.update_user_contract_tree_nodes_ffs = - QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); - let error = require_state_update_record_coverage(&updates, 1, &HashSet::new()) - .expect_err("user-contract leaf without contract-state FFS must fail"); - assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); - assert!(error.to_string().contains("no contract-state FFS"), "{error}"); - require_state_update_record_coverage(&updates, 0, &HashSet::new()) - .expect("genesis may register a user-contract leaf with empty contract-state and IMT"); - let contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 1, - tree_sub_id: 2, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), - }; - updates.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&contract_leaf); - let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); - let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); - let first_next_key = PGoldilocksHash::from_owned_32bytes([0u8; 32]); - let first_leaf = IMTContractStateLeaf:: { - key: first_key, - value: first_value, - next_key: first_next_key, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), - }; - let first_hash = first_leaf.qfhash::(); - updates.update_contract_state_imt_leaves_ffs = - psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 2, 0, &first_hash, &first_key, &first_value, &first_next_key, 0, false, - ) - .to_vec(); - require_state_update_record_coverage(&updates, 1, &HashSet::new()) - .expect("user-contract plus matching contract-state and IMT must close the coverage set"); - } - - #[test] - fn history_ffs_genesis_skips_imt_coverage() { - let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); - let old = tree.get_root(); - let mut registered = empty_updates(old, old); - let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { - tree_id: 262144, - level: 8, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), - }; - registered.update_user_contract_tree_nodes_ffs = - QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); - require_state_update_record_coverage(®istered, 0, &HashSet::new()) - .expect("genesis contract registration has empty contract-state and IMT"); - let registered_error = require_state_update_record_coverage(®istered, 1, &HashSet::new()) - .expect_err("non-genesis registration without contract-state FFS must fail"); - assert!( - registered_error.to_string().contains("no contract-state FFS"), - "{registered_error}" - ); - let mut empty_value = empty_updates(old, old); - let empty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 262144, - tree_sub_id: 0, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), - }; - empty_value.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&empty_contract_leaf); - require_state_update_record_coverage(&empty_value, 0, &HashSet::new()) - .expect("genesis empty contract-state leaf with empty IMT is trusted setup"); - require_state_update_record_coverage(&empty_value, 1, &HashSet::new()) - .expect("new empty contract-state leaf does not require IMT"); - let mut nonempty = empty_updates(old, old); - let nonempty_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 262144, - tree_sub_id: 0, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x11; 32]), - }; - nonempty.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&nonempty_contract_leaf); - require_state_update_record_coverage(&nonempty, 0, &HashSet::new()) - .expect("genesis non-empty contract-state leaf with empty IMT is trusted setup"); - let nonempty_error = require_state_update_record_coverage(&nonempty, 1, &HashSet::from_iter([(262144u64, 0u64, 0u64)])) - .expect_err("updated non-empty contract-state leaf without IMT must fail"); - assert!(nonempty_error.to_string().contains("no IMT record"), "{nonempty_error}"); - replay_state_updates_into_tree::( - &mut tree, &nonempty, 8, 8, 0, 0, &HashSet::new(), - ) - .expect("verify path must honor checkpoint_id=0 and skip IMT pairing"); - let verify_error = replay_state_updates_into_tree::( - &mut SimpleMemoryMerkleRecorderStore::::new(8), - &nonempty, - 8, - 8, - 0, - 1, - &HashSet::from_iter([(262144u64, 0u64, 0u64)]), - ) - .expect_err("verify path at C=1 must still require IMT for a non-empty leaf"); - assert!(verify_error.to_string().contains("no IMT record"), "{verify_error}"); - let mut poisoned = empty_updates(old, old); - poisoned.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; - let width_error = require_state_update_record_coverage(&poisoned, 0, &HashSet::new()) - .expect_err("genesis still rejects poisoned FFS widths"); - assert!(width_error.to_string().contains("user leaf FFS"), "{width_error}"); - } - - #[test] - fn history_ffs_positional_contract_leaf_needs_no_imt() { - let old = PGoldilocksHash::from_owned_32bytes([1u8; 32]); - let mut positional = empty_updates(old, old); - let positional_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 1310720, - tree_sub_id: 0, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0x01; 32]), - }; - positional.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&positional_leaf); - require_state_update_record_coverage(&positional, 1, &HashSet::new()) - .expect("positional non-empty contract-state leaf with no previous IMT entry is exempt"); - let mut cleared = empty_updates(old, old); - let user_contract_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreSingleIdKey { - tree_id: 1, - level: 8, - index: 2, - }, - value: PGoldilocksHash::from_owned_32bytes([0x02; 32]), - }; - cleared.update_user_contract_tree_nodes_ffs = - QMerkleStoreFastSingleNodeSerializer::serialize_single_id_node_to_vec(&user_contract_leaf); - let zero_leaf = parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: 1, - tree_sub_id: 2, - level: 4, - index: 0, - }, - value: PGoldilocksHash::from_owned_32bytes([0u8; 32]), - }; - cleared.update_contract_state_tree_nodes_ffs = - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec(&zero_leaf); - require_state_update_record_coverage(&cleared, 1, &HashSet::new()) - .expect("user-contract pairing survives a contract-state leaf cleared to zero"); - require_state_update_record_coverage(&cleared, 0, &HashSet::from_iter([(1u64, 2u64, 0u64)])) - .expect("genesis skips IMT coverage even with a declared managed set"); - } - - struct IMTPreimageFixture { - next_append: HashMap<(u64, u64), u64>, - } - - #[async_trait::async_trait] - impl psy_node_core::psy_core_db::traits::full::PsyNodeContractStateIMTDatabaseReader for IMTPreimageFixture { - async fn contract_state_imt_get_leaf_preimage( - &self, - _checkpoint_id: u64, - _user_id: u64, - _contract_id: u64, - _leaf_index: u64, - ) -> anyhow::Result>> { - Ok(None) - } - - async fn contract_state_imt_get_leaf_index_for_key( - &self, - _checkpoint_id: u64, - _user_id: u64, - _contract_id: u64, - _key: &PGoldilocksHash, - ) -> anyhow::Result> { - Ok(None) - } - - async fn contract_state_imt_find_predecessor( - &self, - _checkpoint_id: u64, - _user_id: u64, - _contract_id: u64, - _key: &PGoldilocksHash, - ) -> anyhow::Result<(u64, IMTContractStateLeaf)> { - Ok((0, IMTContractStateLeaf::default())) - } - - async fn contract_state_imt_get_next_append_index(&self, user_id: u64, contract_id: u64) -> anyhow::Result { - Ok(self.next_append.get(&(user_id, contract_id)).copied().unwrap_or(0)) - } - } - - fn contract_state_leaf_ffs(user_id: u64, contract_id: u64, index: u64, value_byte: u8) -> Vec { - QMerkleStoreFastDoubleNodeSerializer::serialize_double_id_node_to_vec( - &parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdNode { - key: parth_core::data::hash::merkle_store_key::QMerkleStoreDoubleIdKey { - tree_id: user_id, - tree_sub_id: contract_id, - level: 4, - index, - }, - value: PGoldilocksHash::from_owned_32bytes([value_byte; 32]), - }, - ) - } - - struct CountingHeightStore { - heights: HashMap<(u64, u64), u8>, - calls: std::sync::atomic::AtomicUsize, - batches: std::sync::Mutex)>>, - fail: bool, - } - - impl CountingHeightStore { - fn new(heights: HashMap<(u64, u64), u8>) -> Self { - Self { - heights, - calls: std::sync::atomic::AtomicUsize::new(0), - batches: std::sync::Mutex::new(Vec::new()), - fail: false, - } - } - } - - #[async_trait::async_trait] - impl psy_node_core::psy_core_db::traits::full::PsyNodeCoreDatabaseBasicContractInfoStoreReader< - PGoldilocksFelt, - PGoldilocksHash, - > for CountingHeightStore - { - async fn get_contract_tree_heights( - &self, - checkpoint_id: u64, - contract_ids: &[u64], - ) -> anyhow::Result> { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.batches - .lock() - .expect("height batch log") - .push((checkpoint_id, contract_ids.to_vec())); - if self.fail { - anyhow::bail!("injected height store failure"); - } - Ok(contract_ids - .iter() - .map(|contract_id| self.heights.get(&(checkpoint_id, *contract_id)).copied().unwrap_or(0)) - .collect()) - } - } - - // Same C+I union the verify path feeds the loader: CST groups (1,7),(2,7),(1,8) - // then IMT finals (1,7,0),(1,9,0) with a duplicate (1,7,0) that or_insert keeps first. - const C_AND_I_IDS: [u64; 5] = [7, 7, 8, 7, 9]; - - #[tokio::test] - async fn previous_heights_batch_unique_c_and_i_at_historical_checkpoint() { - let previous = 10u64; - let store = CountingHeightStore::new(HashMap::from([ - ((previous, 7), 8), - ((previous, 8), 16), - ((previous, 9), 24), - ((previous + 1, 7), 32), - ])); - let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>( - &store, - previous, - C_AND_I_IDS, - ) - .await - .expect("batch heights"); - assert_eq!(store.calls.load(std::sync::atomic::Ordering::SeqCst), 1); - let batches = store.batches.lock().expect("height batch log"); - assert_eq!(batches.as_slice(), &[(previous, vec![7, 8, 9])]); - drop(batches); - assert_eq!(require_previous_contract_height(&heights, 7).unwrap(), 8); - assert_eq!(require_previous_contract_height(&heights, 8).unwrap(), 16); - assert_eq!(require_previous_contract_height(&heights, 9).unwrap(), 24); - let later = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous + 1, [7]) - .await - .expect("later checkpoint is a different key"); - assert_eq!(require_previous_contract_height(&later, 7).unwrap(), 32); - } - - #[tokio::test] - async fn previous_heights_reject_zero_missing_and_injected_db_error() { - let previous = 10u64; - let store = CountingHeightStore::new(HashMap::from([((previous, 7), 0)])); - let heights = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&store, previous, [7, 8]) - .await - .expect("missing maps to zero without a store error"); - let zero = require_previous_contract_height(&heights, 7).expect_err("zero height must reject"); - assert!(zero.to_string().contains("height is zero"), "{zero}"); - let missing = require_previous_contract_height(&heights, 8).expect_err("absent height must reject"); - assert!(missing.to_string().contains("height is zero"), "{missing}"); - let failing = CountingHeightStore { - fail: true, - ..CountingHeightStore::new(HashMap::new()) - }; - let error = load_previous_contract_heights::<_, PGoldilocksFelt, PGoldilocksHash>(&failing, previous, [7]) - .await - .expect_err("store failure must reject"); - assert!(error.to_string().contains("MissingAuthenticatedState"), "{error}"); - assert!(error.to_string().contains("injected height store failure"), "{error}"); - assert_eq!(failing.calls.load(std::sync::atomic::Ordering::SeqCst), 1); - } - - - #[tokio::test] - async fn history_imt_managed_set_follows_tree_append_index() { - let mut managed_tree = IMTPreimageFixture { next_append: HashMap::new() }; - managed_tree.next_append.insert((1, 2), 2); - - // A first-time insert on an IMT-managed tree is NOT exempt: the is_new_key - // index write is mandatory, so the new slot must ship its IMT record too. - let mut new_key = empty_updates( - PGoldilocksHash::from_owned_32bytes([1u8; 32]), - PGoldilocksHash::from_owned_32bytes([2u8; 32]), - ); - new_key.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 5, 0x07); - let managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &new_key) - .await - .unwrap(); - assert_eq!(managed, HashSet::from_iter([(1u64, 2u64, 5u64)])); - let error = require_state_update_record_coverage(&new_key, 1, &managed) - .expect_err("new key on a managed tree without IMT record must fail"); - assert!(error.to_string().contains("no IMT record"), "{error}"); - - let mut tracked_changed = empty_updates( - PGoldilocksHash::from_owned_32bytes([1u8; 32]), - PGoldilocksHash::from_owned_32bytes([2u8; 32]), - ); - tracked_changed.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x05); - let tracked_managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &tracked_changed) - .await - .unwrap(); - assert_eq!(tracked_managed, HashSet::from_iter([(1u64, 2u64, 0u64)])); - - let positional_tree = IMTPreimageFixture { next_append: HashMap::new() }; - let untracked = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&positional_tree, &tracked_changed) - .await - .unwrap(); - assert!(untracked.is_empty(), "trees with no IMT entries are positional"); - require_state_update_record_coverage(&tracked_changed, 1, &untracked) - .expect("collector output for a positional leaf must pass coverage"); - - let mut cleared = empty_updates( - PGoldilocksHash::from_owned_32bytes([1u8; 32]), - PGoldilocksHash::from_owned_32bytes([2u8; 32]), - ); - cleared.update_contract_state_tree_nodes_ffs = contract_state_leaf_ffs(1, 2, 0, 0x00); - let cleared_managed = imt_managed_leaves_from_db::<_, PGoldilocksFelt, PGoldilocksHash>(&managed_tree, &cleared) - .await - .unwrap(); - assert!(cleared_managed.is_empty(), "leaf cleared to zero keeps the no-IMT behavior"); - } - - - #[test] - fn history_imt_terminal_next_key_must_be_zero() { - let first_key = PGoldilocksHash::from_owned_32bytes([0x22; 32]); - let first_value = PGoldilocksHash::from_owned_32bytes([0x33; 32]); - let nonzero_next = PGoldilocksHash::from_owned_32bytes([0x99; 32]); - let leaf = IMTContractStateLeaf:: { - key: first_key, - value: first_value, - next_key: nonzero_next, - next_index: parth_core::felt::FromPrimitiveValuesFelt::from_u64_value(0), - }; - let leaf_hash = leaf.qfhash::(); - let entry = psy_data::v1::qdata::contract::serialize_imt_leaf_ffs_entry_v2( - 1, 0, 3, &leaf_hash, &first_key, &first_value, &nonzero_next, 0, false, - ); - let error = require_imt_leaf_ffs_consistency::(&entry) - .expect_err("terminal IMT next_index=0 with nonzero next_key must fail"); - assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); - assert!(error.to_string().contains("terminal next_key"), "{error}"); - } - - #[test] - fn history_poison_included_transition() { - let mut tree = SimpleMemoryMerkleRecorderStore::::new(8); - let old = tree.get_root(); - let mut updates = empty_updates(old, old); - updates.update_user_leaves_ffs = vec![0u8; PSY_OBJECT_FFS_SIZE_USER_LEAF + 1]; - let error = replay_state_updates_into_tree::( - &mut tree, &updates, 8, 8, 0, 1, &HashSet::new(), - ) - .expect_err("poisoned user-leaf width must fail"); - assert!(error.to_string().contains("InvalidStateUpdates"), "{error}"); - assert!(!error.to_string().contains("auto")); - } - - fn sample_store_object(old_root: [u8; 32], new_root: [u8; 32], salt: u8) -> (psy_data::p2p::Proposal, Vec) { - sample_store_object_at(old_root, new_root, salt, 99, 1) - } - - fn sample_store_object_at( - old_root: [u8; 32], - new_root: [u8; 32], - salt: u8, - base_checkpoint_id: u64, - proposer_sub_id: u16, - ) -> (psy_data::p2p::Proposal, Vec) { - let output = vec![salt; psy_data::p2p::MAX_FINALIZER_OUTPUT_BYTES]; - let proof = vec![0xABu8; 32]; - let mut state_updates = vec![0u8; 40 + 64 + 20]; - state_updates[40..72].copy_from_slice(&old_root); - state_updates[72..104].copy_from_slice(&new_root); - let worker_tag = [0x11u8; 32]; - let body = psy_data::p2p::encode_proposal_body(&output, &proof, &state_updates, &worker_tag).unwrap(); - let proposal = psy_data::p2p::proposal_from_parts( - 1, - 0, - base_checkpoint_id, - proposer_sub_id, - [salt; 32], - psy_data::p2p::sha256(&output), - psy_data::p2p::sha256(&proof), - psy_data::p2p::sha256(&state_updates), - psy_data::p2p::sha256(&body), - ); - (proposal, body) - } - - #[tokio::test] - async fn history_ready_store_selection() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (proposal, body) = sample_store_object([1u8; 32], [2u8; 32], 1); - store.save_proposal(&proposal, &body).await.unwrap(); - let found = store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); - assert_eq!(found.len(), 1); - assert_eq!(found[0].proposal_id, proposal.proposal_id); - let loaded = store - .load_proposal(&[1u8; 32], &[2u8; 32]) - .await - .unwrap() - .expect("slot holds the stored pair"); - assert_eq!(loaded.0.proposal_id, proposal.proposal_id); - assert_eq!(loaded.1, body); - } - - #[tokio::test] - async fn history_ready_single_body_per_pair() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (first, first_body) = sample_store_object([9u8; 32], [8u8; 32], 3); - let (second, second_body) = sample_store_object([9u8; 32], [8u8; 32], 4); - store.save_proposal(&first, &first_body).await.unwrap(); - store.save_proposal(&second, &second_body).await.unwrap(); - let found = store.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); - assert_eq!(found.len(), 1); - assert_eq!(found[0].proposal_id, second.proposal_id); - let found = store.lookup_transition(&[9u8; 32], &[8u8; 32]).await.unwrap(); - assert_eq!(found[0].proposal_id, second.proposal_id); - } - - fn test_node(seed: u8) -> NodeId { - let mut raw = [0u8; 38]; - raw[..6].copy_from_slice(&[0x00, 0x24, 0x08, 0x01, 0x12, 0x20]); - raw[6..].fill(seed); - NodeId::from_raw(raw).unwrap() - } - - #[derive(Clone, Copy, Debug)] - enum PeerFault { - Timeout, - NotAValidator(NodeId), - Closed, - } - - impl PeerFault { - fn to_error(self) -> crate::realm::network::NetworkError { - use crate::realm::network::NetworkError; - match self { - PeerFault::Timeout => NetworkError::Timeout("test fault".to_string()), - PeerFault::NotAValidator(node) => NetworkError::NotAValidator(node), - PeerFault::Closed => NetworkError::CommandChannelClosed, - } - } - } - - #[derive(Default)] - struct PeerDoubleState { - offers: std::collections::HashMap>, - bodies: std::collections::HashMap<(NodeId, [u8; 32]), Result, PeerFault>>, - faults: std::collections::HashMap, - looked_up: tokio::sync::Mutex>, - inflight: std::sync::atomic::AtomicUsize, - max_inflight: std::sync::atomic::AtomicUsize, - } - - impl PeerDoubleState { - fn offer(mut self, peer: NodeId, lookup: RealmTransition, proposal: Proposal) -> Self { - self.offers.entry(peer).or_default().push((lookup, proposal)); - self - } - - fn body(mut self, peer: NodeId, proposal_id: [u8; 32], body: Result, PeerFault>) -> Self { - self.bodies.insert((peer, proposal_id), body); - self - } - - fn failing(mut self, peer: NodeId, fault: PeerFault) -> Self { - self.faults.insert(peer, fault); - self - } - } - - /// A command channel answering lookups and body ranges from a fixed inventory. - fn spawn_peer_double(state: std::sync::Arc) -> RealmNetworkCommands { - use crate::realm::network::{NetworkError, RealmNetworkCommand}; - let (commands, mut rx) = tokio::sync::mpsc::channel(64); - tokio::spawn(async move { - while let Some(command) = rx.recv().await { - match command { - RealmNetworkCommand::LookupProposal { - destination, - request, - response, - } => { - let live = state - .inflight - .fetch_add(1, std::sync::atomic::Ordering::SeqCst) - + 1; - state - .max_inflight - .fetch_max(live, std::sync::atomic::Ordering::SeqCst); - let answered = match state.faults.get(&destination) { - Some(fault) => Err(fault.to_error()), - None => { - let mut looked = state.looked_up.lock().await; - let offers = state.offers.get(&destination); - let mut entries = Vec::with_capacity(request.pairs.len()); - for pair in &request.pairs { - looked.insert((pair.old_root, pair.new_root)); - let candidates = offers - .map(|offers| { - offers - .iter() - .filter(|(lookup, _)| *lookup == *pair) - .map(|(_, proposal)| proposal.clone()) - .take(PROPOSAL_LOOKUP_CANDIDATES_PER_PAIR) - .collect() - }) - .unwrap_or_default(); - entries.push(ProposalLookupEntry { - transition: *pair, - candidates, - }); - } - Ok(ProposalLookupResponse::candidates(entries)) - } - }; - let _ = response.send(answered); - state - .inflight - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - } - RealmNetworkCommand::RequestBody { - destination, - request, - response, - } => { - let answered = match state.bodies.get(&(destination, request.proposal_id)) { - Some(Ok(body)) => { - let start = request.offset as usize; - let take = (request.max_bytes as usize) - .min(body.len().saturating_sub(start)); - Ok(BodyChunkResponse { - offset: request.offset, - data: body[start..start + take].to_vec(), - eof: start + take == body.len(), - body_len: body.len() as u64, - body_hash: sha256(body), - }) - } - Some(Err(fault)) => Err(fault.to_error()), - None => Err(NetworkError::CommandChannelClosed), - }; - let _ = response.send(answered); - } - _ => {} - } - } - }); - RealmNetworkCommands::from_channel(commands, test_node(200)) - } - - fn lookup_of(old_root: [u8; 32], new_root: [u8; 32]) -> RealmTransition { - RealmTransition { old_root, new_root } - } - - fn single_peer(seed: u8) -> (Vec<(u16, NodeId)>, NodeId) { - let peer = test_node(seed); - (vec![(1, peer)], peer) - } - - async fn promote_all(store: &ProposalStore, outcomes: Vec) -> usize { - let mut promoted = 0usize; - for outcome in outcomes { - if let TransitionFetchOutcome::Staged(_, staged) = outcome { - store.install(staged).await.unwrap(); - promoted += 1; - } - } - promoted - } - - #[tokio::test] - async fn history_window_stages_each_offered_pair() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (first, first_body) = sample_store_object([1u8; 32], [2u8; 32], 1); - let (second, second_body) = sample_store_object([2u8; 32], [3u8; 32], 2); - let first_lookup = lookup_of([1u8; 32], [2u8; 32]); - let second_lookup = lookup_of([2u8; 32], [3u8; 32]); - let (members, peer) = single_peer(1); - let client = spawn_peer_double(std::sync::Arc::new( - PeerDoubleState::default() - .offer(peer, first_lookup, first.clone()) - .offer(peer, second_lookup, second.clone()) - .body(peer, first.proposal_id, Ok(first_body)) - .body(peer, second.proposal_id, Ok(second_body)), - )); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let outcomes = stage_transition_blocks( - &client, - &store, - &peers, - 1, - 0, - &[first_lookup, second_lookup], - &[], - ) - .await; - assert_eq!(outcomes.len(), 2); - assert!(store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap().is_empty()); - assert_eq!(promote_all(&store, outcomes).await, 2); - assert_eq!( - store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap()[0].proposal_id, - first.proposal_id - ); - assert_eq!( - store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap()[0].proposal_id, - second.proposal_id - ); - } - - #[tokio::test] - async fn history_window_pair_failure_does_not_poison_sibling() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (good, good_body) = sample_store_object([4u8; 32], [5u8; 32], 5); - let (bad, mut bad_body) = sample_store_object([5u8; 32], [6u8; 32], 6); - bad_body[0] ^= 0xFF; - let good_lookup = lookup_of([4u8; 32], [5u8; 32]); - let bad_lookup = lookup_of([5u8; 32], [6u8; 32]); - let (members, peer) = single_peer(3); - let client = spawn_peer_double(std::sync::Arc::new( - PeerDoubleState::default() - .offer(peer, good_lookup, good.clone()) - .offer(peer, bad_lookup, bad.clone()) - .body(peer, good.proposal_id, Ok(good_body)) - .body(peer, bad.proposal_id, Ok(bad_body)), - )); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let outcomes = stage_transition_blocks( - &client, - &store, - &peers, - 1, - 0, - &[good_lookup, bad_lookup], - &[], - ) - .await; - assert!(matches!(outcomes[0], TransitionFetchOutcome::Staged(..))); - assert!(matches!(outcomes[1], TransitionFetchOutcome::Failed(..))); - assert_eq!(promote_all(&store, outcomes).await, 1); - assert_eq!(store.lookup_transition(&[4u8; 32], &[5u8; 32]).await.unwrap().len(), 1); - assert!(store.lookup_transition(&[5u8; 32], &[6u8; 32]).await.unwrap().is_empty()); - } - - #[tokio::test] - async fn history_lookup_failure_switches_to_backup_peer() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (proposal, body) = sample_store_object([7u8; 32], [8u8; 32], 7); - let lookup = lookup_of([7u8; 32], [8u8; 32]); - let primary = test_node(4); - let backup = test_node(5); - let members = vec![(1, primary), (2, backup)]; - let client = spawn_peer_double(std::sync::Arc::new( - PeerDoubleState::default() - .failing(primary, PeerFault::Timeout) - .offer(backup, lookup, proposal.clone()) - .body(backup, proposal.proposal_id, Ok(body)), - )); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &[lookup], &[]).await; - assert_eq!(promote_all(&store, outcomes).await, 1); - assert_eq!( - store.lookup_transition(&[7u8; 32], &[8u8; 32]).await.unwrap()[0].proposal_id, - proposal.proposal_id - ); - } - - #[tokio::test] - async fn history_window_fetch_concurrency_and_coverage() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (members, peer) = single_peer(8); - let mut state = PeerDoubleState::default(); - let mut needed = Vec::new(); - for i in 0..5u8 { - let old = [i; 32]; - let new = [i + 1; 32]; - let (proposal, mut body) = sample_store_object(old, new, 20 + i); - if i == 2 { - body[0] ^= 0xFF; - } - let lookup = lookup_of(old, new); - state = state - .offer(peer, lookup, proposal.clone()) - .body(peer, proposal.proposal_id, Ok(body)); - needed.push(lookup); - } - let state = std::sync::Arc::new(state); - let client = spawn_peer_double(state.clone()); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &needed, &[]).await; - assert_eq!(promote_all(&store, outcomes).await, 4); - assert!( - state.max_inflight.load(std::sync::atomic::Ordering::SeqCst) <= PROPOSAL_LOOKUP_CONCURRENCY - ); - let looked = state.looked_up.lock().await; - for lookup in &needed { - assert!( - looked.contains(&(lookup.old_root, lookup.new_root)), - "missing lookup pair=({},{})", - hex::encode(lookup.old_root), - hex::encode(lookup.new_root) - ); - } - drop(looked); - assert_eq!(store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap().len(), 0); - assert_eq!(store.lookup_transition(&[0u8; 32], &[1u8; 32]).await.unwrap().len(), 1); - assert_eq!(store.lookup_transition(&[3u8; 32], &[4u8; 32]).await.unwrap().len(), 1); - } - - #[tokio::test] - async fn history_window_across_epochs() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let epoch = CHECKPOINTS_PER_EPOCH; - let first_proposal_count = epoch - 1; - let second_proposal_count = epoch * 3 - 1; - let (first, first_body) = - sample_store_object_at([1u8; 32], [2u8; 32], 30, first_proposal_count, 1); - let (second, second_body) = - sample_store_object_at([2u8; 32], [3u8; 32], 31, second_proposal_count, 2); - let first_lookup = lookup_of([1u8; 32], [2u8; 32]); - let second_lookup = lookup_of([2u8; 32], [3u8; 32]); - let supplier = test_node(9); - let members = vec![(3, supplier)]; - let client = spawn_peer_double(std::sync::Arc::new( - PeerDoubleState::default() - .offer(supplier, first_lookup, first.clone()) - .offer(supplier, second_lookup, second.clone()) - .body(supplier, first.proposal_id, Ok(first_body)) - .body(supplier, second.proposal_id, Ok(second_body)), - )); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let outcomes = stage_transition_blocks( - &client, - &store, - &peers, - 1, - 0, - &[first_lookup, second_lookup], - &[], - ) - .await; - assert_eq!(promote_all(&store, outcomes).await, 2); - assert_eq!(first.proposer_sub_id, 1); - assert_eq!(second.proposer_sub_id, 2); - let first_found = store.lookup_transition(&[1u8; 32], &[2u8; 32]).await.unwrap(); - let second_found = store.lookup_transition(&[2u8; 32], &[3u8; 32]).await.unwrap(); - assert_eq!(first_found[0].proposal_id, first.proposal_id); - assert_eq!(second_found[0].proposal_id, second.proposal_id); - assert!(first.base_checkpoint_id < second.base_checkpoint_id); - } - - #[tokio::test] - async fn history_window_without_anchor_leaf_fails_closed() { - let error = CatchupPeers::select(&[], 1).expect_err("empty occupied leaves must fail closed"); - assert!( - error - .to_string() - .contains("no other validator peer at this checkpoint"), - "{error}" - ); - } - - #[tokio::test] - async fn history_empty_answer_marks_every_pair_absent() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (members, _peer) = single_peer(10); - let state = std::sync::Arc::new(PeerDoubleState::default()); - let client = spawn_peer_double(state.clone()); - let peers = CatchupPeers::select(&members, 9).unwrap(); - let needed = vec![lookup_of([8u8; 32], [9u8; 32])]; - let outcomes = stage_transition_blocks(&client, &store, &peers, 1, 0, &needed, &[]).await; - assert!(matches!(outcomes[0], TransitionFetchOutcome::Absent(..))); - assert!(state.looked_up.lock().await.contains(&([8u8; 32], [9u8; 32]))); - } - - #[tokio::test] - async fn history_resend_revotes() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (proposal, body) = sample_store_object([3u8; 32], [4u8; 32], 3); - store.save_proposal(&proposal, &body).await.unwrap(); - store.save_proposal(&proposal, &body).await.unwrap(); - let first = store - .read_body_chunk(&BodyChunkRequest { - proposal_id: proposal.proposal_id, - offset: 0, - max_bytes: 64, - }) - .await - .unwrap(); - store.save_proposal(&proposal, &body).await.unwrap(); - let second = store - .read_body_chunk(&BodyChunkRequest { - proposal_id: proposal.proposal_id, - offset: 0, - max_bytes: 64, - }) - .await - .unwrap(); - assert_eq!(first.body_hash, proposal.body_hash); - assert_eq!(second.body_hash, proposal.body_hash); - assert_eq!(first.body_len, second.body_len); - } - - #[tokio::test] - async fn history_invalid_candidate_ab() { - let dir = tempfile::tempdir().unwrap(); - let store = ProposalStore::open(dir.path()).await.unwrap(); - let (candidate_b, body_b) = sample_store_object([10u8; 32], [11u8; 32], 40); - let (mut candidate_x, body_x) = sample_store_object([10u8; 32], [12u8; 32], 41); - candidate_x.base_checkpoint_id = candidate_b.base_checkpoint_id; - store.save_proposal(&candidate_b, &body_b).await.unwrap(); - store.save_proposal(&candidate_x, &body_x).await.unwrap(); - let found = store.lookup_transition(&[10u8; 32], &[12u8; 32]).await.unwrap(); - assert_eq!(found.len(), 1); - assert_eq!(found[0].proposal_id, candidate_x.proposal_id); - let sibling = store.lookup_transition(&[10u8; 32], &[11u8; 32]).await.unwrap(); - assert_eq!(sibling[0].proposal_id, candidate_b.proposal_id); - let absent = store.lookup_transition(&[11u8; 32], &[10u8; 32]).await.unwrap(); - assert!(absent.is_empty()); - assert!(dir - .path() - .join("bodies") - .join(format!("{}_{}", hex::encode([10u8; 32]), hex::encode([11u8; 32]))) - .exists()); - } -} From ac78ed2a83ee1b6bfdce5fb5a485cb1928f21d38 Mon Sep 17 00:00:00 2001 From: 0x8f701 Date: Fri, 18 Sep 2026 17:25:05 +0800 Subject: [PATCH 3/3] Delete UnappliedTransition; scan returns Option. - first_root_change returns Option<(RealmTransition, included C)>. - Identity last-modified C is accounted at apply_proposal_ffs from the same list. - Hop predicate stays new_root != old_root; fetch path unchanged. --- docs/src/dev/TERMINOLOGY.md | 8 +-- .../src/realm/processor/catchup.rs | 69 +++++-------------- .../src/realm/processor/core/process_block.rs | 14 ++-- 3 files changed, 29 insertions(+), 62 deletions(-) diff --git a/docs/src/dev/TERMINOLOGY.md b/docs/src/dev/TERMINOLOGY.md index c4b63901..e0cc5e6c 100644 --- a/docs/src/dev/TERMINOLOGY.md +++ b/docs/src/dev/TERMINOLOGY.md @@ -71,9 +71,9 @@ Realm commit durable mappings, FFS rows, then set_latest_checkpoint_i | Apply | Realm (or gatherer FastForward) | Execute the FFS / tree transition. Pair of **unapplied**. | The checkpoint marker write. | | Commit | Local processor | `commit_state` writes records and the marker. Adjective: **last_committed_**. | Coordinator inclusion. A proposal that is only certified. | -`included` appears as `included_checkpoint` on `UnappliedTransition`. That is the coordinator -checkpoint that already carries the transition. It is not a good function -adjective: the ready-path function *applies* FFS, then *commits* locally. +`included` is the coordinator checkpoint that already carries the transition +(`first_root_change` returns it with the `RealmTransition`). It is not a good +function adjective: the ready-path function *applies* FFS, then *commits* locally. Rejected names for the ready-path function: @@ -84,7 +84,7 @@ Rejected names for the ready-path function: | `apply_committed_proposal_ffs` | `committed` is `last_committed_*`, which this function produces, not consumes. | Chosen name: **`apply_proposal_ffs`**. Verb `apply`, object `proposal_ffs`, same -family as `apply_history_proposal` and `first_unapplied_transition`. +family as `apply_history_proposal` and `first_root_change`. Catch-up sibling: `apply_history_transitions` walks `C+1..=tip`. The ready path applies the next unapplied proposal FFS once per `sync_and_verify`. diff --git a/psy_node_common/src/realm/processor/catchup.rs b/psy_node_common/src/realm/processor/catchup.rs index 63ba28ec..6bcb7c6f 100644 --- a/psy_node_common/src/realm/processor/catchup.rs +++ b/psy_node_common/src/realm/processor/catchup.rs @@ -23,36 +23,27 @@ pub enum TransitionFetchOutcome { Failed(RealmTransition, anyhow::Error), } -/// Next coordinator-authenticated realm transition after `last_committed`. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum UnappliedTransition { - None { accounted_checkpoint: u64 }, - Real { transition: RealmTransition, included_checkpoint: u64 }, -} - /// Last-modified events must be chronological. Equal roots are leaf rewrites, not bodies. /// A→B→A keeps the A→B hop because that first value differs. -pub(crate) fn first_unapplied_transition( +pub(crate) fn first_root_change( last_committed: u64, last_committed_root: [u8; 32], last_modifieds: &[(u64, [u8; 32])], -) -> UnappliedTransition { +) -> Option<(RealmTransition, u64)> { let mut old_root = last_committed_root; - let mut accounted_checkpoint = last_committed; for &(checkpoint_id, new_root) in last_modifieds { if checkpoint_id <= last_committed { continue; } if new_root != old_root { - return UnappliedTransition::Real { - transition: RealmTransition { old_root, new_root }, - included_checkpoint: checkpoint_id, - }; + return Some(( + RealmTransition { old_root, new_root }, + checkpoint_id, + )); } - accounted_checkpoint = checkpoint_id; old_root = new_root; } - UnappliedTransition::None { accounted_checkpoint } + None } /// Peer set chosen once per catch-up batch: one primary and at most one backup. @@ -342,55 +333,29 @@ mod tests { const ROOT_B: [u8; 32] = [0xB; 32]; #[test] - fn first_unapplied_transition_skips_identity_rewrite() { - let outcome = first_unapplied_transition(15, ROOT_A, &[(16, ROOT_A)]); - assert_eq!( - outcome, - UnappliedTransition::None { - accounted_checkpoint: 16 - } - ); + fn first_root_change_skips_identity_rewrite() { + assert_eq!(first_root_change(15, ROOT_A, &[(16, ROOT_A)]), None); } #[test] - fn first_unapplied_transition_keeps_cycle_first_hop() { - let outcome = first_unapplied_transition(15, ROOT_A, &[(20, ROOT_B), (50, ROOT_A)]); + fn first_root_change_keeps_cycle_first_hop() { assert_eq!( - outcome, - UnappliedTransition::Real { - transition: RealmTransition { - old_root: ROOT_A, - new_root: ROOT_B, - }, - included_checkpoint: 20, - } + first_root_change(15, ROOT_A, &[(20, ROOT_B), (50, ROOT_A)]), + Some((RealmTransition { old_root: ROOT_A, new_root: ROOT_B }, 20)), ); } #[test] - fn first_unapplied_transition_skips_identity_then_takes_real() { - let outcome = first_unapplied_transition(15, ROOT_A, &[(16, ROOT_A), (50, ROOT_B)]); + fn first_root_change_skips_identity_then_takes_real() { assert_eq!( - outcome, - UnappliedTransition::Real { - transition: RealmTransition { - old_root: ROOT_A, - new_root: ROOT_B, - }, - included_checkpoint: 50, - } + first_root_change(15, ROOT_A, &[(16, ROOT_A), (50, ROOT_B)]), + Some((RealmTransition { old_root: ROOT_A, new_root: ROOT_B }, 50)), ); } #[test] - fn first_unapplied_transition_empty_is_accounted_at_committed() { - let outcome = first_unapplied_transition(15, ROOT_A, &[]); - assert_eq!( - outcome, - UnappliedTransition::None { - accounted_checkpoint: 15 - } - ); + fn first_root_change_empty_is_none() { + assert_eq!(first_root_change(15, ROOT_A, &[]), None); } fn build_proposal_with_body(old_root: [u8; 32], new_root: [u8; 32], salt: u8) -> (psy_data::p2p::Proposal, Vec) { diff --git a/psy_node_common/src/realm/processor/core/process_block.rs b/psy_node_common/src/realm/processor/core/process_block.rs index 3d8ab786..2fdbf385 100644 --- a/psy_node_common/src/realm/processor/core/process_block.rs +++ b/psy_node_common/src/realm/processor/core/process_block.rs @@ -268,21 +268,23 @@ where } last_modifieds.reverse(); let old_root = self.db.state.last_committed_realm_end_root.into_owned_32bytes(); - let (transition, included_checkpoint_id) = match crate::realm::processor::catchup::first_unapplied_transition( + let (transition, included_checkpoint_id) = match crate::realm::processor::catchup::first_root_change( self.db.state.last_committed_checkpoint_id, old_root, &last_modifieds, ) { - crate::realm::processor::catchup::UnappliedTransition::None { accounted_checkpoint } => { - if accounted_checkpoint > self.db.state.last_committed_checkpoint_id { + None => { + if let Some(&(accounted_checkpoint, _)) = last_modifieds + .iter() + .rev() + .find(|(checkpoint_id, _)| *checkpoint_id > self.db.state.last_committed_checkpoint_id) + { self.db.state.last_committed_checkpoint_id = accounted_checkpoint; self.db.shared_state.update_from_core_state(&self.db.state).await?; } return self.db.sync_to_coordinator_checkpoint_id(latest_checkpoint_id).await; } - crate::realm::processor::catchup::UnappliedTransition::Real { transition, included_checkpoint } => { - (transition, included_checkpoint) - } + Some((transition, included_checkpoint)) => (transition, included_checkpoint), }; coordinator_realm_state = self.db.coordinator_client .rc_get_realm_root_and_last_modified_checkpoint(included_checkpoint_id, self.db.state.realm_id_u64)