diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3483253..6de915e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} - name: Download canonical app deps - run: just -f examples/canonical-app/justfile download-deps + run: just canonical download-deps - name: Run guest tests run: just canonical-test-guest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83a0de4..023a864 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# Release workflow: git tag v* builds sequencer tarballs, CM images, and pushes +# Release workflow: git tag v* builds wallet-sequencer tarballs, CM images, and pushes # watchdog OCI images to ghcr.io + docker.io (requires DOCKERHUB_USERNAME / # DOCKERHUB_TOKEN org or repo secrets, same as cartesi/cli). @@ -25,8 +25,8 @@ permissions: packages: write jobs: - build-sequencer: - name: Build sequencer (${{ matrix.arch }}) + build-wallet-sequencer: + name: Build wallet-sequencer (${{ matrix.arch }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -70,8 +70,10 @@ jobs: with: tool: cross + # `sequencer` is a library crate; the runnable reference binary is the + # `wallet-sequencer` example app (sequencer lib + the placeholder wallet). - name: Build (release) - run: cross build -p sequencer --release --locked --target ${{ matrix.target }} + run: cross build -p wallet-sequencer --release --locked --target ${{ matrix.target }} - name: Package env: @@ -82,14 +84,14 @@ jobs: set -euo pipefail mkdir -p dist - mkdir -p "package/sequencer-${TAG}-linux-${ARCH}" - cp "target/${TARGET}/release/sequencer" "package/sequencer-${TAG}-linux-${ARCH}/sequencer" + mkdir -p "package/wallet-sequencer-${TAG}-linux-${ARCH}" + cp "target/${TARGET}/release/wallet-sequencer" "package/wallet-sequencer-${TAG}-linux-${ARCH}/wallet-sequencer" bash scripts/generate-release-manifest.sh \ --tag "${TAG}" \ --git-sha "${GITHUB_SHA}" \ - --output "package/sequencer-${TAG}-linux-${ARCH}/RELEASE.json" + --output "package/wallet-sequencer-${TAG}-linux-${ARCH}/RELEASE.json" - cat > "package/sequencer-${TAG}-linux-${ARCH}/RUNNING.md" <<'EOF' + cat > "package/wallet-sequencer-${TAG}-linux-${ARCH}/RUNNING.md" <<'EOF' ## Running Required environment variables: @@ -106,17 +108,17 @@ jobs: CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \ - ./sequencer + ./wallet-sequencer ``` EOF - tar -C package -czf "dist/sequencer-${TAG}-linux-${ARCH}.tar.gz" "sequencer-${TAG}-linux-${ARCH}" + tar -C package -czf "dist/wallet-sequencer-${TAG}-linux-${ARCH}.tar.gz" "wallet-sequencer-${TAG}-linux-${ARCH}" - name: Upload artifact uses: actions/upload-artifact@v6 with: - name: sequencer-linux-${{ matrix.arch }} - path: dist/sequencer-*.tar.gz + name: wallet-sequencer-linux-${{ matrix.arch }} + path: dist/wallet-sequencer-*.tar.gz build-canonical-machine-image: name: Build canonical machine images @@ -140,7 +142,7 @@ jobs: cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} - name: Download canonical app deps - run: just -f examples/canonical-app/justfile download-deps + run: just canonical download-deps - name: Build machine image (devnet) run: just canonical-build-machine-image @@ -230,7 +232,7 @@ jobs: name: Publish GitHub Release runs-on: ubuntu-latest needs: - - build-sequencer + - build-wallet-sequencer - build-canonical-machine-image - build-watchdog-image steps: @@ -244,7 +246,7 @@ jobs: uses: actions/download-artifact@v6 with: path: dist - pattern: "{sequencer-linux-*,canonical-machine-images}" + pattern: "{wallet-sequencer-linux-*,canonical-machine-images}" - name: Flatten artifacts env: diff --git a/AGENTS.md b/AGENTS.md index f3138c9..7c628f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,26 @@ The sequencer must advance `safe_block` honestly. If it freezes `safe_block` (to Under honest sequencer operation and no infrastructure outages, soft confirmations match the canonical order. This is an **optimistic guarantee** — the sequencer is predicting a future the scheduler has not yet computed. When the sequencer goes offline, submits stale batches, or tries to censor direct inputs, the scheduler's force-drain backstop kicks in and the affected soft confirmations become invalid. +### Where the duality lives in code (change one, check all) + +The precise acceptance algorithm — decode → sender → nonce → structural → +staleness → frame execution order → nonce advance — is **owned by +[`docs/protocol/scheduler-semantics.md`](docs/protocol/scheduler-semantics.md)**. +This section is the map. + +Scheduler-acceptance semantics exist in exactly three implementations that must agree: + +1. the canonical fold — `Scheduler` ([`sequencer-core/src/scheduler/mod.rs`](sequencer-core/src/scheduler/mod.rs)), the same source compiled into the on-chain machine and driven bare-metal by the recovery fold; +2. the off-chain acceptance predicate — `ProtocolTiming::scheduler_accepts` ([`sequencer-core/src/protocol.rs`](sequencer-core/src/protocol.rs)), which feeds `safe_accepted_batches`; +3. the inclusion lane's live prediction (drain + execution order). + +The expected-nonce fold is homed next to `scheduler_accepts` as `advance_expected_batch_nonce` (same file); the submitter's `decide_submit_start` consumes it, and `populate_safe_accepted_batches` keeps a deliberate inline copy (its advance is interleaved with storage-only side effects — the R2 content-identity check and the divergence freeze — that can't move below the protocol layer). Touching any of these means re-checking the others — their agreement is the system's most load-bearing invariant (see [`docs/invariants.md`](docs/invariants.md)). + +Two mechanical facts the agreement rests on: + +- **Drain attribution.** At a safe-frontier advance, the newly-drained directs are sequenced into the **new** frame — the frame stamped with the **new** `safe_block`. So frame K's wire content reads "directs ≤ S_K, then user ops validated on top of them", exactly the scheduler's drain-before-ops rule. +- **Empty batches are never stale and consume the nonce** (no first frame to measure staleness against). Consistent across all implementations, test-pinned. + ## Batch Staleness and Recovery ### Staleness @@ -74,19 +94,22 @@ If a batch is stale, all existing subsequent batches are also invalid. The sched Rather than waiting for a batch to go stale on L1, the sequencer uses a **danger threshold** (`MAX_WAIT_BLOCKS − MARGIN`). The threshold is *only a trigger*: it tells the system "stop running, hand off to recovery." It does not encode "this batch is doomed" — that decision belongs to the post-flush cascade. -The cycle crosses a process boundary by design: - -1. **Detector trips + process exits** — the in-process [`DangerDetector`](sequencer/src/recovery/detector.rs) polls `Storage::check_danger` on a cadence. When the L1-view-stale, observed closed-batch, observed Tip, or batch-relative wall-clock arm fires, the detector exits with `DetectorExit::RecoveryRequired`, the runtime maps that to `RunError::DangerDetected`, and the process exits with a non-zero status. Stopping the process is how the sequencer goes offline: no more user-op acceptance, no more batch submission. -2. **Orchestrator respawns** — systemd/k8s/etc. restarts the process. -3. **Startup syncs and dispatches** — the fresh process syncs the L1 safe head if reachable, re-runs `Storage::check_danger`, then [`decide_startup_action`](sequencer/src/recovery/mod.rs) chooses the startup path. -4. **Startup runs recovery** — dispatched by the danger status: - - **`RecoverTip`** → [`Storage::recover_aging_tip(danger_threshold)`](sequencer/src/storage/recovery.rs): no flush ran. The open Tip has no L1 footprint, so invalidate it directly once its first frame has aged past the danger threshold. - - **`FlushAndCascade`** → [`MempoolFlusher`](sequencer/src/recovery/flusher.rs) consumes pending wallet-nonce slots, startup re-syncs L1, then [`Storage::recover_post_flush(danger_threshold)`](sequencer/src/storage/recovery.rs) cascades from the first non-gold closed batch (every non-gold batch past the post-flush gold frontier is doomed — Silver-stale, Silver-poisoned, or no-op'd Pending). If all closed batches landed gold, fall through to a Tip check against `danger_threshold` (handles the corner case where `S_tip = S_closed`, the closed batch lands fresh, and the Tip's age clears the danger zone after the flush wait). - - **`Proceed`** → [`Storage::recover_aging_tip(danger_threshold)`](sequencer/src/storage/recovery.rs): no flush ran and no danger was detected. Closed batches past gold may still be in their natural lifecycle, so leave them alone; the Tip check is defensive and normally a no-op. - - **`Refuse`** → startup stops and surfaces the reason to the operator. Refusal is used when the L1 safe block timestamp is missing/too old, or when batch-relative wall-clock estimation says unresolved work has consumed its remaining runway without observed safe-state support for recovery. -5. **Normal operation resumes** — the lane, submitter, input reader, and a fresh detector all start up. - -See [`docs/recovery/README.md`](docs/recovery/README.md) Step 5 for the "everything past gold is doomed" mental model and why the post-flush cascade is unconditional rather than threshold-based. +The cycle crosses a process boundary by design: the in-process +[`DangerDetector`](sequencer/src/recovery/detector.rs) polls +`Storage::check_danger` on a cadence and **exits the process** when any +non-`Safe` arm fires (stopping the process is how the sequencer goes offline); +the orchestrator respawns; startup syncs the L1 safe head, re-runs +`check_danger`, and [`decide_startup_action`](sequencer/src/recovery/mod.rs) +dispatches — `Proceed` (no recovery writes), `RecoverTip` (invalidate the aging +Tip directly; it has no L1 footprint, so no flush), `FlushAndCascade` (flush +every wallet-nonce slot, re-sync, cascade everything past the gold frontier), +or `Refuse` (surface to the operator). Then normal operation resumes. + +The authoritative dispatch table, the "everything past gold is doomed" model, +and the per-path rationale live in +[`docs/recovery/README.md`](docs/recovery/README.md) — that document **owns** +the recovery design; this section is only the map. Do not restate dispatch +details here. ### Detection: safe-only, with wall-clock fallback @@ -104,7 +127,7 @@ See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the full mo - **Trusted:** InputBox contract, our own Ethereum node (fail-stop, not byzantine), operator config, batch-submitter key. - **Adversarial:** `POST /tx` callers, direct-input senders, the L1 mempool and block builders (zombie transactions are a first-class threat). -- **Semi-trusted, fail-stop:** fallback RPC providers (Infura / Alchemy). +- **RPC endpoint:** single (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`), trusted fail-stop, **must be one consistent node** — no fallback tier exists yet (see the threat model's actor table). - **Self-trust:** the sequencer trusts its own code is correct. Bugs that emit malformed batches are fault states requiring manual intervention, not threats to defend against at runtime. - **In scope:** correctness bugs *and* exploitation. Under rollup semantics, a correctness bug that causes scheduler/sequencer state divergence is as severe as direct theft. @@ -114,9 +137,10 @@ Top-level layout follows the system's data flow. Each sequencer module correspon ### Workspace -- `sequencer/` — main sequencer binary and library. +- `sequencer/` — sequencer **library** (no binary). App crates compose it into a binary. - `sequencer-core/` — shared domain types (`Application`, `SignedUserOp`, `SequencedL2Tx`, `Batch`, `Frame`). - `examples/app-core/` — placeholder wallet app implementing the `Application` trait. +- `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. The model for what an app author builds (their `Application` impl ≙ `app-core`; their binary crate ≙ this). - `examples/canonical-app/` — on-chain scheduler reference implementation. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. @@ -124,10 +148,10 @@ Top-level layout follows the system's data flow. Each sequencer module correspon ### Sequencer module layout -- `sequencer/src/main.rs` — thin binary entrypoint. -- `sequencer/src/lib.rs` — public sequencer API (`run`, `RunConfig`). +- `sequencer/src/lib.rs` — public sequencer API. The thin binary entrypoints live in `examples/wallet-sequencer/`. +- `sequencer/src/harness.rs` — CLI harness: the `setup`/`run`/`flush-mempool` subcommand parser, `dispatch`, and the R4 exit-code projection. An app's `main` is ~5 lines (`run_main` + a genesis-app closure). - `sequencer/src/http.rs` — shared HTTP error type, JSON `ErrorResponse`, `ApiConfig`, and `axum::serve` orchestration. -- `sequencer/src/runtime/` — process bootstrap, `RunConfig`, EIP-712 domain, `ShutdownSignal`, shared `clock::unix_now_ms`. +- `sequencer/src/runtime/` — process orchestration: `setup` (phase A — pin identity, initial sync, genesis snapshot, `setup_complete` marker), `run` (phase B — boot workers from a set-up DB), `flush` (`flush-mempool`), plus `config`, `error` (incl. exit-code projection), `shutdown`, shared `clock::unix_now_ms`, and the `workers` lifecycle. - `sequencer/src/ingress/` — public write path. - `api.rs` — `POST /tx` handler, JSON-rejection mapping. - `inclusion_lane/` — single-lane hot-path loop (`mod.rs`), catch-up replay, config, error types. @@ -140,7 +164,7 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - `provider.rs` — alloy provider construction. - `partition.rs` — long-block-range retry helper. - `sequencer/src/recovery/` — preemptive recovery startup procedure (`mod.rs`), runtime danger detector (`detector.rs`), and mempool flusher (`flusher.rs`). -- `sequencer/src/storage/` — SQLite persistence, split by writer role (`ingress`, `egress`, `l1_inputs`, `l1_submission`, `recovery`, `admin`, plus shared `mod`, `open`, `internals`, and `migrations/`). +- `sequencer/src/storage/` — SQLite persistence, split by writer role (`ingress`, `egress`, `l1_inputs`, `l1_submission`, `recovery`, `admin`, `safe_accepted_batches`, `snapshot_dumps`, plus shared `mod`, `open`, `convert`, `queries`, `mutations`, and `migrations/`). ## Key Concepts @@ -159,7 +183,7 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - API validates the EIP-712 signature and enqueues a `SignedUserOp`. Method payload decoding happens during application execution, not at ingress. - **Deposits are direct-input-only** (L1 → L2) and must not be represented as user ops. -- Rejections (`InvalidNonce`, `InvalidMaxFee`, `InsufficientGasBalance`) produce no state mutation and are not persisted. +- Rejections (`InvalidNonce`, `InvalidMaxFee`, `InsufficientFeeBalance`) produce no state mutation and are not persisted. These are protocol-level rejection semantics every app must implement: nonces prevent user-op replay, fees prevent spam against the sequencer's DA budget. ("Fee", not "gas" — the fee tracks DA; compute metering, if it ever exists, is a separate future concept.) - Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `safe_inputs`, and `sequenced_l2_txs`. Recovery metadata lives in `safe_accepted_batches`; batch lifecycle state (sealed/invalidated) lives on the `batches` row itself as write-once timestamps. - Frame fee is persisted in `frames.fee` and is fixed for the lifetime of that frame. The next frame's fee is sampled from `batch_policy_derived.recommended_fee` at rotation. - Wallet state (balances, nonces) is in-memory today — not persisted. @@ -175,7 +199,7 @@ Top-level layout follows the system's data flow. Each sequencer module correspon ## Application Trait Contract -Implementors of the `Application` trait must respect these contracts. The sequencer assumes them without runtime enforcement. +Implementors of the `Application` trait must respect these contracts. The sequencer assumes them without runtime enforcement. The full, code-grounded contract — method table, dump round-trip durability, the safe-block clock — is **owned by [`docs/protocol/application-contract.md`](docs/protocol/application-contract.md)**; the essentials follow. ### Replay determinism @@ -187,11 +211,15 @@ The sequencer persists every included user op and every ingested direct input. O ### No implicit state -Application state changes must flow exclusively through `execute_valid_user_op` and `execute_direct_input`. Mutating state from `validate_user_op` or `current_user_nonce` breaks replay determinism. +Application state changes must flow exclusively through `execute_valid_user_op` and `execute_direct_input`. Mutating state from `validate_user_op` breaks replay determinism. + +### One execution entry point + +User ops are executed only through `sequencer_core::application::validate_and_execute_user_op` (a free function, deliberately not an overridable trait method): it enforces the protocol-level `max_fee >= current_fee` guard before app validation, so no `Application` impl can skip it. Both the inclusion lane and the canonical scheduler call it — part of the duality agreement. ## Hot-Path Invariants -- API ack is tied to chunk durability, not frame/batch closure. +- API ack is tied to chunk durability, not frame/batch closure. "Durable" means power-loss-durable: WAL with `synchronous=FULL`, so every commit fsyncs before anything externalizes on it (review R3). - Chunk commit and ack remain low-latency; frame closure is orthogonal and can happen less frequently. - `POST /tx` queue admission: `try_send` on a full queue returns `429 OVERLOADED` with message `queue full`. - Frame closure happens when direct inputs are drained, and also whenever batch closure happens. @@ -200,6 +228,17 @@ Application state changes must flow exclusively through `execute_valid_user_op` ## Storage Invariants +Writer roles — one writer per table; reads over batch data go through the `valid_*` views: + +| Writer | Writes | +|---|---| +| inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `sequenced_l2_txs`, `dumps`/`pending_snapshots` (batch close), `finalized_snapshot` (promotion) | +| input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `deployment_identity`, `canonical_divergence` (poison marker, review R2) | +| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, scoped `pending_snapshots` clear, `wallet_nonce_watermark` (flush no-ops, write-before-broadcast) | +| batch submitter | `wallet_nonce_watermark` (write-before-broadcast, review R1a — its only write) | +| egress (HTTP) | `dumps.lease_count` (leases) | +| admin | `batch_policy` | + - Storage model is append-oriented; avoid mutable status flags for open/closed entities. - Open batch/frame are derived by "latest row" convention. - A frame's leading direct-input prefix is derivable from `sequenced_l2_txs` plus `frames.safe_block`. @@ -209,7 +248,7 @@ Application state changes must flow exclusively through `execute_valid_user_op` - Included user-op identity is tracked by application nonce logic; no DB uniqueness constraint (removed to allow resubmission after recovery). - **Reads over batch data go through `valid_batches`, `valid_closed_batches`, `valid_open_batch`, and `valid_sequenced_l2_txs` views.** These encapsulate the "exclude invalidated rows" filter so individual queries don't repeat it. Writers go to the base tables. - **`batches` row columns partition cleanly by writer.** `sealed_at_ms` is owned by the inclusion lane (set when closing a batch); `invalidated_at_ms` is owned by recovery (set during cascade). Each is write-once (NULL → non-NULL, never back) and enforced by triggers. The partial unique index `ux_single_valid_tip` guarantees at most one row has both NULL — the Tip. -- The inclusion lane is the **only writer** of open batch/frame state. `Storage::append_user_ops_chunk` and the `close_*` methods trust the in-memory `WriteHead`; FK + PK constraints catch the dangerous failure modes. +- The inclusion lane is the **only writer** of open batch/frame state. `Storage::append_user_ops_chunk` and the `close_*` methods trust the in-memory `WriteHead`; the Tip-targeting triggers and the `pos_in_frame` PK catch stale-`WriteHead` bugs for **user ops**. **Direct-input sequencing has no structural uniqueness guard** (re-drain support requires duplicate `safe_input_index` across invalidated batches) — double-sequencing prevention rests on the lane's drain-cursor discipline and its startup re-derivation (see [`docs/invariants.md`](docs/invariants.md)). ## Type Boundaries @@ -221,35 +260,35 @@ Application state changes must flow exclusively through `execute_valid_user_op` ## HTTP Endpoints - **Ingress** (public-facing): `POST /tx`. -- **Egress** (internal indexers/watchdog): `GET /ws/subscribe`, `GET /finalized_state`, `GET /finalized_state/inclusion_block`, `GET /latest_snapshot`, `GET /livez`, `GET /readyz`, `GET /healthz`. +- **Egress** (internal indexers/watchdog): `GET /ws/subscribe`, `GET /finalized_state`, `GET /finalized_state/inclusion_block`, `GET /latest_snapshot`, `GET /livez`, `GET /readyz`, `GET /healthz`. The snapshot/state endpoints are **operator-only** (no auth) and must not be exposed publicly; the streaming routes hold a GC lease for the response lifetime ([`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)). Today both sides serve from one listener; the planned API split puts each side on its own port (same binary) so internal probes and subscribers can be firewalled from public submit traffic. -`/ws/subscribe` internal guardrails: subscriber cap 64, catch-up cap 50000. When the catch-up window is exceeded, the handler upgrades and then closes with WebSocket close code `1008` (`POLICY`), reason `catch-up window exceeded`. - -Health semantics: `/livez` — 200 if the process is alive. `/readyz` — 200 if shutdown not requested AND inclusion-lane channel still open, else 503. `/healthz` — JSON `{ status, inclusion_lane }` mirroring the same 200/503. - -Snapshot endpoints (`/finalized_state`, `/finalized_state/inclusion_block`, `/latest_snapshot`) are **operator-only** (no auth) — they serve the watchdog and indexers and must not be exposed publicly. The two streaming routes hold a GC lease on the dump for the response lifetime, released even on client disconnect (via a drop-guard); `Storage::reset_dump_leases` at startup is the crash backstop. Shapes are in [`README.md`](README.md); dump format in [`docs/snapshots/format.md`](docs/snapshots/format.md) and the snapshot lifecycle (take/promote/GC/lease, crash-safety) in [`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md). +Message shapes, caps, close codes, and health semantics are **owned by [`README.md`](README.md)** (the API contract) — do not restate them here. ## Environment Variables -**Required:** +Split by subcommand (the phase split). **`setup`** (required): - `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` - `CARTESI_SEQUENCER_BLOCKCHAIN_ID` - `CARTESI_SEQUENCER_APP_ADDRESS` -- `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` +- `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS` (the submitter address — `setup` is L1-read-only and never signs). **Must be a dedicated address**: `setup`'s detection gate refuses if the submitter's wallet nonce is unsettled, so reusing a busy address (e.g. the contract deployer, whose deploy-tx tail isn't safe at setup time) false-positives. The devnet uses anvil account 9 (`DEVNET_SEQUENCER_ADDRESS`), distinct from the account-0 deployer. +- `CARTESI_SEQUENCER_CHECKPOINT_BLOCK` (optional, default `0` = genesis) — the trusted checkpoint machine's L1 inclusion block. `setup` refuses (typed `SetupRefuse`, exit 40 = run `setup --recovery`) if a previous instance left work past it. PR3 detects only; loading a non-genesis checkpoint machine is `setup --recovery` (PR5). -**Optional:** +**`run`** (required) — chain id / app address / submitter address are read from the DB `setup` pinned, not from args: -- `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`) -- `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data`; DB file `sequencer.db` inside it) -- `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` -- `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS` (default 5000) -- `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH` (default 2) -- `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default 300, ~1h at 12s/block) -- `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default derived before the danger threshold) -- `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default 12) +- `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` +- `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` + +**Optional** (names only — defaults and semantics are **owned by +[`sequencer/src/runtime/config.rs`](sequencer/src/runtime/config.rs)**; a +defaults list here drifted once already): `CARTESI_SEQUENCER_HTTP_ADDR`, `CARTESI_SEQUENCER_DATA_DIR`, +`CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, +`CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`, `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS`, +`CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (fixed default, independent of the margin; +must be strictly below the danger threshold or startup refuses), +`CARTESI_SEQUENCER_SECONDS_PER_BLOCK`. ## Coding Conventions @@ -258,7 +297,7 @@ Snapshot endpoints (`/finalized_state`, `/finalized_state/inclusion_block`, `/la - Surface user-facing errors via `ApiError` (in `http.rs`); keep internal failures descriptive but safe. - Avoid introducing heavy dependencies without strong reason. - Documentation style: lean. Module headers (1–4 lines) + docs on public methods only when the contract isn't obvious from name+signature. Use inline comments for **why**, never for **what**. -- **Don't layer defense-in-depth checks against sequencer self-bugs.** Correctness is enforced via tests and review. See "Self-trust" in [`docs/threat-model/README.md`](docs/threat-model/README.md). +- **Impossible states fail loud; they are never handled.** Cheap cross-module assertions of *real invariants* are encouraged (assert, trigger `RAISE`, typed error) — a loud crash is recoverable by design; silent divergence is not. Never add graceful fallbacks, neighbor re-validation, or silent absorbers (`INSERT OR IGNORE`, saturating decode of impossible data) for states the contracts rule out; and an assertion must check a real invariant, never an environmental assumption (clock monotonicity is the cautionary tale). Decision test and rationale: [`docs/invariants.md`](docs/invariants.md); trust boundaries: "Self-trust" in [`docs/threat-model/README.md`](docs/threat-model/README.md). ## Testing Guidance @@ -286,14 +325,20 @@ cargo fmt --all cargo clippy --all-targets --all-features -- -D warnings ``` -Run server: +Run server (two phases — `setup` once, then `run`; see `README.md` "Running"): ```bash +# setup (L1-read-only; takes the submitter ADDRESS, not the key) CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ +CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 \ +cargo run -p wallet-sequencer -- setup + +# run (keyed; reads identity from the set-up DB) +CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \ -cargo run -p sequencer +cargo run -p wallet-sequencer -- run ``` ## Always / Ask First / Never @@ -305,6 +350,7 @@ cargo run -p sequencer - Add or update tests when logic changes. - Run at least `cargo check` before finishing. - Read `docs/recovery/` before touching recovery code, and `docs/threat-model/` before touching trust-boundary code. +- Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing, and the latest review ledger under [`docs/review/`](docs/review/) for known-open findings in the code you're about to touch. ### Ask First @@ -338,8 +384,11 @@ Before finishing a change, ensure: ## Related Documents -- [`README.md`](README.md) — product framing, user-facing trust model. +- [`README.md`](README.md) — product framing, user-facing trust model, **API contract** (endpoint shapes, caps, close codes, health semantics). - [`CLAUDE.md`](CLAUDE.md) — shell setup, quick reference, pointer back here. +- [`docs/protocol/`](docs/protocol/) — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (the canonical acceptance algorithm, I1) and [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` FFI trait contract). +- [`docs/invariants.md`](docs/invariants.md) — register of cross-module invariants (what's load-bearing across files) + the fail-loud check policy. +- [`docs/review/`](docs/review/) — dated correctness-review ledgers; open findings, settled designs, work packages. - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. - [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history. - [`docs/snapshots/`](docs/snapshots/) — app snapshots: [`format.md`](docs/snapshots/format.md) (dump trait + wire format) and [`lifecycle.md`](docs/snapshots/lifecycle.md) (take/promote/GC/lease design + crash-safety). diff --git a/CLAUDE.md b/CLAUDE.md index aa9131e..b4ae18c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,9 +30,10 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en ## Workspace Layout -- `sequencer/` — main sequencer binary and library. +- `sequencer/` — sequencer library (no binary; app crates build the binary). - `sequencer-core/` — shared domain types consumed by both sequencer and scheduler. - `examples/app-core/` — placeholder wallet app implementing `Application`. +- `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. - `examples/canonical-app/` — on-chain scheduler reference implementation. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. @@ -53,6 +54,9 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en ## Before You Start Real Work - **[`AGENTS.md`](AGENTS.md)** — mission, requirements, invariants, duality, recovery, conventions, rules. +- **[`docs/protocol/`](docs/protocol/)** — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (canonical acceptance algorithm) and [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` FFI trait). Read before touching the scheduler, the gold frontier, the fold, or an `Application` impl. +- **[`docs/invariants.md`](docs/invariants.md)** — cross-module invariants register + the fail-loud check policy. Check it before changing anything it lists as load-bearing. +- **[`docs/review/`](docs/review/)** — dated correctness-review ledgers: known-open findings, settled designs, work packages. Check for open findings in code you're about to touch. - **[`docs/threat-model/README.md`](docs/threat-model/README.md)** — trust boundaries and in-scope threats. - **[`docs/recovery/README.md`](docs/recovery/README.md)** — preemptive recovery design + TLA+ proofs. - **[`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)** — snapshot lifecycle design + invariants (take/promote/GC, crash-safety). Read before touching the inclusion lane's safe-frontier/snapshot path. diff --git a/Cargo.lock b/Cargo.lock index fad2b8f..3539857 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1286,9 +1286,7 @@ name = "canonical-app" version = "0.1.0" dependencies = [ "alloy-primitives", - "alloy-sol-types", "app-core", - "ethereum_ssz", "k256", "sequencer-core", "trolley", @@ -3191,7 +3189,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.8+spec-1.1.0", ] [[package]] @@ -3889,9 +3887,9 @@ dependencies = [ "tokio", "tokio-tungstenite", "tokio-util", + "toml", "tower-http", "tracing", - "tracing-subscriber", ] [[package]] @@ -3902,6 +3900,7 @@ dependencies = [ "alloy-sol-types", "ethereum_ssz", "ethereum_ssz_derive", + "k256", "ruint", "serde", "thiserror 1.0.69", @@ -3973,6 +3972,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4499,6 +4507,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "1.1.0+spec-1.1.0" @@ -4508,6 +4537,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.8+spec-1.1.0" @@ -4515,7 +4558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" dependencies = [ "indexmap 2.13.0", - "toml_datetime", + "toml_datetime 1.1.0+spec-1.1.0", "toml_parser", "winnow 1.0.0", ] @@ -4529,6 +4572,12 @@ dependencies = [ "winnow 1.0.0", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.3" @@ -4791,6 +4840,16 @@ dependencies = [ "libc", ] +[[package]] +name = "wallet-sequencer" +version = "0.1.0" +dependencies = [ + "app-core", + "sequencer", + "tokio", + "tracing-subscriber", +] + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 9bcd353..b99284b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,12 @@ members = [ "examples/app-core", "examples/canonical-app", "examples/canonical-test", + "examples/wallet-sequencer", "tests/benchmarks", "tests/harness", "tests/e2e", ] -default-members = ["sequencer"] +default-members = ["sequencer", "examples/wallet-sequencer"] [workspace.package] version = "0.1.0" diff --git a/README.md b/README.md index 95cd4e6..f00aff4 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,14 @@ The sequencer is a **centralized, single-writer** system. It cannot steal funds **Direct inputs** (L1 → L2 messages, used for deposits) bypass the sequencer entirely. They are posted directly to L1 and are **uncensorable** by the sequencer — the scheduler drains them at every `safe_block` boundary. A censoring sequencer can delay when a direct input is executed (up to `MAX_WAIT_BLOCKS`, ~4h), but cannot prevent it. +Soft confirmations are an **optimistic prediction**: the sequencer also +cross-checks every batch the scheduler accepts on L1 against the batch it +sealed locally (a content-identity check), and refuses to operate further the +moment they differ. Detection happens when the divergent batch reaches L1 +*safe* finality, so soft confirmations issued inside that window (~2 L1 +epochs) can be built on already-diverged state — an inherent, bounded +property of the optimistic model. + The third case is handled by the recovery subsystem. Batches that are too old when they reach L1 (`inclusion_block − safe_block ≥ MAX_WAIT_BLOCKS`) are skipped by the scheduler. This "staleness" poisons the nonce counter: all subsequent batches become unreachable regardless of their individual freshness. The sequencer detects this via a danger-zone threshold, preemptively goes offline, flushes the L1 mempool, and cascade-invalidates the doomed chain. See [`docs/recovery/`](docs/recovery/) for the full design, TLA+ formal verification, and design history. The sequencer trusts its own code is bug-free. Recovery means recovery from liveness failures, which can legitimately happen even in the absence of bugs (infrastructure outages, network failures, gateway failure). Code-level bugs are a separate problem handled by tests and review. See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the complete threat model applied across the codebase. @@ -63,17 +71,36 @@ The batch submitter posts closed batches to L1's InputBox contract. Each batch c ## Running +The sequencer runs in two phases. **`setup`** pins the +deployment identity, does the initial L1 sync, and registers the genesis +snapshot — run it once. It is L1-read-only: it takes the batch-submitter +*address*, never the signing key. **`run`** boots the sequencer from the +set-up DB, reading identity from it (so chain id / app address are not `run` +arguments); it holds the signing key because it submits. + ```bash +# Phase A — set up the data dir (run once; idempotent). CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ +CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 \ +cargo run -p wallet-sequencer -- setup + +# Phase B — run the sequencer. +CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac09...f2ff80 \ -cargo run -p sequencer +cargo run -p wallet-sequencer -- run ``` -Required: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `_FILE`). +A third subcommand, **`flush-mempool`**, settles the batch-submitter wallet +nonce on demand (keyed operator tool). + +`setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`. +`run` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `_FILE`); it refuses to boot until `setup` has completed. + +Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`. -Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`. +Process exit codes follow the R4 orchestrator contract: `0` clean shutdown, `10` restart (expect a recovery boot), `20` transient refusal (retry with backoff), `30` terminal (operator required — e.g. setup not complete, identity mismatch, canonical divergence), `1`/`101` unclassified/panic. Fixed protocol identity (EIP-712): @@ -129,7 +156,7 @@ Message shapes: ``` ```json -{ "kind": "direct_input", "offset": 11, "payload": "0x..." } +{ "kind": "direct_input", "offset": 11, "sender": "0x...", "block_number": 123, "payload": "0x..." } ``` Success response: @@ -174,8 +201,8 @@ released even on client disconnect. ## Project Layout -- `sequencer/src/main.rs`: thin binary entrypoint -- `sequencer/src/lib.rs`: public crate surface (`run`, `RunConfig`) +- `sequencer/src/lib.rs`: public crate surface (`run`, `RunConfig`) — the sequencer is a library; app crates build the binary (see `examples/wallet-sequencer/`) +- `examples/wallet-sequencer/`: binary crate composing the sequencer library with the placeholder wallet app - `sequencer/src/http.rs`: shared HTTP error type, JSON error shape, and `axum::serve` orchestration - `sequencer/src/runtime/`: process bootstrap, config parsing, EIP-712 domain, shutdown signal, shared clock - `sequencer/src/ingress/`: public write path — `POST /tx` (`api.rs`) and the inclusion lane (`inclusion_lane/`: hot-path loop, chunk/frame/batch rotation, catch-up, snapshot lifecycle) diff --git a/docs/invariants.md b/docs/invariants.md new file mode 100644 index 0000000..302a490 --- /dev/null +++ b/docs/invariants.md @@ -0,0 +1,282 @@ +# Cross-Module Invariants + +The register of invariants whose **statement, enforcement, and consumers live in +different files**. Single-file invariants belong in that file's comments; this +file exists because the most dangerous knowledge in this codebase is the +invariant that spans modules with nothing pinning it — the kind a locally-sound +refactor silently breaks. + +Each entry: what holds → where it's enforced → who depends on it → what breaks. +Symbol names can drift; verify against the code before relying on an entry. +When you change anything listed under *enforced by*, re-check every line under +*depended on by*. + +## The check policy (fail-loud) + +**Impossible states fail loud; they are never handled.** + +- An invariant violation gets exactly one response: abort the operation loudly + (assert, trigger `RAISE`, typed error). Cheap cross-module assertions at + boundaries are *encouraged*: a loud crash is recoverable by design + (orchestrator respawn + startup recovery), while a silently-tolerated bug + that externalizes (a signed batch, an ack, a feed event) is state divergence + — theft-equivalent and unrecoverable at runtime. +- **Never handle gracefully what cannot happen.** No fallback branches, no + re-deriving a neighbor's answer to double-check it, no `Option`-handling for + can't-be-`None`. One contract, one source of truth, no second code path. +- **Never absorb silently.** No `INSERT OR IGNORE`, saturating decode, or + `unwrap_or_default` on data the contracts make impossible; use the loud + variant of the same operation. +- An assertion must check a **real invariant** — true in every legitimate + execution, including crash-recovery, replays, and clock steps — never an + environmental assumption. (Cautionary tale: `sealed_at_ms >= created_at_ms` + was CHECK-enforced, wall-clock regression is legitimate, and the constraint + wedged recovery — review F8.) + +Decision test for any proposed check: (a) real invariant? (b) near-zero cost? +(c) fails loud with no alternative code path? Three yeses → write it. Any no → +don't. + +## Register + +### I1. Scheduler-acceptance semantics agree across all implementations + +- **Authoritative prose:** [`docs/protocol/scheduler-semantics.md`](protocol/scheduler-semantics.md). +- **Holds:** the canonical fold (`Scheduler`, + `sequencer-core/src/scheduler/mod.rs`), the off-chain predicate + (`ProtocolTiming::scheduler_accepts`, `sequencer-core/src/protocol.rs`), and + the inclusion lane's live prediction produce the same + accept/reject/ordering decisions for every input. (Known, documented + exception: the predicate omits the two structural rejections — self-trust, + since the simulator only runs over the sequencer's own well-formed batches; + the omission is documented in `scheduler-semantics.md`, not currently + test-pinned.) +- **Enforced by:** review + tests only. No mechanism. +- **Depended on by:** everything — the gold frontier, recovery's cascade pivot, + promotion, soft-confirmation honesty. +- **Breaks:** silent permanent scheduler/sequencer divergence. +- The expected-nonce fold is homed next to `scheduler_accepts` as + `advance_expected_batch_nonce`; `decide_submit_start` consumes it, while + `populate_safe_accepted_batches` keeps a deliberate inline copy (its advance + interleaves with storage-only side effects that can't move below the protocol + layer — see the call-site comment). + +### I2. Drain attribution: drained directs land in the new frame + +- **Holds:** at a safe-frontier advance, the newly-drained directs are + sequenced into the **new** frame, which is stamped with the **new** + `safe_block` (`close_frame_in`, `storage/ingress.rs`). Frame K's wire content + is therefore "directs ≤ S_K, then ops validated on top". +- **Enforced by:** `close_frame_in` ordering; lane convention. +- **Depended on by:** the duality (scheduler's drain-before-ops equals the + flattened replay order); catch-up; the feed. +- **Breaks:** ops validated against a state the scheduler won't reproduce — + divergence. + +### I3. Frame `safe_block`s are non-decreasing along the spine + +- **Holds:** every frame opens at the current safe frontier, which only + advances. +- **Enforced by:** lane flow + `append_safe_inputs`' monotonicity asserts + (`storage/l1_inputs.rs`). +- **Depended on by:** `check_danger`'s arm ordering (see I4); the scheduler's + within-batch monotonicity check; "if the frontier batch is fresh, all are". +- **Breaks:** I4's guarantee evaporates; danger detection mis-orders. + +### I4. Tip-only cascade ⇒ every closed batch is gold + +- **Holds:** `check_danger` checks `ClosedBatchInDanger` **before** + `TipInDanger`; with I3, the closed frontier is always at least as old as the + Tip, so the Tip arm can only fire when no non-gold closed batch exists. +- **Enforced by:** the arm order in `Storage::check_danger` + (`storage/recovery.rs`) + I3. +- **Depended on by:** the dispatch table's meaning (a `RecoverTip` boot may + skip the flush *because* nothing closed is doomed). **No longer load-bearing + for the pending clear**: since the F9 fix (2026-06-11) the clear is scoped + to `nonce >= pivot.nonce` in `cascade_and_reopen`, so a valid in-flight + closed batch's pending survives any cascade by construction, regardless of + arm order. +- **Breaks:** a Tip-only cascade while a closed batch is doomed would leave + the doomed batch un-cascaded until the next detector cycle (liveness lag, + not the old crash-loop). + +### I5. Pending-clear is scoped to the cascade and runs in its transaction + +- **Holds:** recovery deletes only pending rows with `nonce >= + pivot.nonce`, atomically with the cascade and the full-backlog tip reopen + (`cascade_and_reopen`, `storage/recovery.rs`). +- **Enforced by:** the single `write` tx + the scoped + `clear_pending_dumps_from_nonce_in`. +- **Depended on by:** catch-up never loading a cascaded batch's state + (cleared rows), and promotion never hitting a deleted row for a batch that + stayed valid (surviving rows) — the `lifecycle.md` §6/§8 wedge is + unrepresentable. +- **Breaks:** widening the delete re-arms the promote-wedge crash-loop; + narrowing it lets catch-up resume from a cascaded batch's state. + +### I6. A committed promotion implies an advanced drain + +- **Holds:** promotion is folded into the drain's transaction + (`close_frame_only_promoting`). +- **Enforced by:** the single `write` tx in `storage/ingress.rs`; the + standalone `Storage::promote_finalized` is test-only by policy. +- **Depended on by:** crash-safety of the safe-frontier walk + (`lifecycle.md` §5–§6). +- **Breaks:** restart re-processes the range and re-promotes a deleted pending + row — crash-loop. + +### I7. A committed batch close has a promotable pending row + +- **Holds:** seal + next-Tip open + `pending_snapshots` insert commit together + (`close_frame_and_batch_with_pending_dump`). +- **Enforced by:** single transaction; `create_dump` happens before, on disk. +- **Depended on by:** promotion (`promote_finalized_in` hard-fails on a missing + row). +- **Breaks:** promotion wedge at the sealed batch's landing. + +### I8. Always-load: a finalized snapshot and a valid Tip exist before the lane starts + +- **Holds:** cold start registers the genesis dump as finalized and opens the + genesis Tip; recovery reopens the Tip atomically across cascades. +- **Enforced by:** `Workers::spawn` order (`ensure_finalized_snapshot`, + `ensure_open_tip`) + recovery's in-tx reopen. +- **Depended on by:** catch-up's unconditional load path + (`CatchUpError::NoSnapshot` is fail-loud, not a branch); the lane's + `NoOpenTip` fail-loud load. +- **Breaks:** startup crash (loud — by design). + +### I9. Acceptance identity: "accepted nonce N" means "our valid batch N" + +- **Holds:** by nonce **and content** since WP3 (review R2, 2026-06-12): every + fully-accepted landing is compared against the local valid closed batch at + that nonce — `keccak256(landed bytes)` vs the hash stamped at seal by the + same encode path the submitter broadcasts. +- **Enforced by:** prevention — the flush resolving every wallet-nonce slot + before a cascade reuses a nonce, anchored by the persisted watermark (I14); + detection — the content-identity check in + `populate_safe_accepted_batches`, which on violation persists the + `canonical_divergence` marker and freezes the frontier (I15). +- **Depended on by:** the gold frontier, cascade pivot selection, promotion, + local-state ↔ canonical-state agreement. +- **Breaks:** was silent divergence (review F1's zombie, F3's power-loss + re-seal); now a detected `CanonicalDivergence` refusal whose remedy is + cockroach recovery. + +### I10. Replay-offset sentinel: `0` means "from genesis" + +- **Holds:** `valid_ordered_l2_tx_head` returns 0 on an empty stream, and + catch-up pages with `offset > cursor` — sound because `sequenced_l2_txs` + rowids start at 1 and rows are never deleted (invalidated rows are filtered, + not removed), so offsets are globally increasing and 0 is never a real + offset. +- **Enforced by:** SQLite rowid semantics + append-only convention. +- **Depended on by:** catch-up, the feed cursor, snapshot `l2_tx_index`. +- **Breaks:** first transaction skipped or double-applied on replay. + +### I11. Own-batch safe inputs are sequenced but never executed or fanned out + +- **Holds:** batch-submitter-sent safe inputs enter `sequenced_l2_txs` like any + drained input, but are skipped by sender at catch-up replay + (`catch_up.rs`), at live execution (`execute_safe_inputs_chunk`), and at WS + delivery (feed filter). +- **Enforced by:** sender checks at each consumer (three places — keep them in + sync). +- **Depended on by:** replay correctness (a batch payload must never execute as + a deposit); feed consumers' state. +- **Breaks:** batch bytes applied as a direct input — divergence. + +### I12. Safe head advances only on real observation; `synced_at_ms` is genuine progress time + +- **Holds:** the reader early-returns when the fetched head doesn't advance; + `append_safe_inputs` asserts monotonicity and stamps `synced_at_ms` only on + commit. +- **Enforced by:** reader floor check + asserts (`storage/l1_inputs.rs`). +- **Depended on by:** the wall-clock danger arms (`L1ViewStale`, + `EstimatedBatchInDanger`) — their baseline must be true progress time, or + outages are masked. +- **Breaks:** danger detection silently late during exactly the outages it + exists for. + +### I13. No `dumps` row points at a missing directory + +- **Holds:** file create (fsync'd) before row insert; row delete before file + delete; orphan *files* are acceptable and swept at startup. +- **Enforced by:** ordering split between `storage/snapshot_dumps.rs` + (SQLite-only) and the lane's FS half (`inclusion_lane/snapshot.rs`) — the + module boundary *is* the ordering guarantee. +- **Depended on by:** `from_dump` at catch-up; the serving endpoints. +- **Breaks:** resume crash-loop. (Power-loss caveat until review R3 lands: + a non-fsynced row delete can rewind past a completed unlink — review F4.) + +### I14. Watermark ≥ wallet nonce of every tx ever broadcast + +- **Holds:** since WP2 (review R1a, 2026-06-11) — the watermark commits + durably (`synchronous=FULL`) before any broadcast at a new nonce, + uniformly for batch txs and flush no-ops. +- **Enforced by:** write-before-broadcast — `EthereumBatchPoster::submit_batches` + raises through `WalletNonceWatermarkSink` before its first send; + `MempoolFlusher::flush_and_wait` likewise before its no-ops, and refuses to + complete until `safe >= watermark + 1`. +- **Depended on by:** flush completeness, TLA+ Implementation Constraint 1, + cascade soundness (I9). +- **Breaks:** zombie txs evade the flush — review F1. + +### I15. Divergence marker present ⇒ acceptance frontier frozen + +- **Holds:** since WP3 (review R2, 2026-06-12). A fully-accepted landing that + fails the content-identity check writes the `canonical_divergence` + singleton **in the same transaction** as the sync that detected it, and + `populate_safe_accepted_batches` returns early whenever the marker exists — + so no acceptance row, no promotion, and no gold-frontier advance can ever + happen past a detected divergence. +- **Enforced by:** the marker guard at the top of + `populate_safe_accepted_batches` + `check_danger`'s first arm + (`CanonicalDivergence`, ranked ahead of every other arm) + the + `Refuse(CanonicalDivergence)` startup dispatch. +- **Depended on by:** standard recovery never running on a diverged frontier + (a flush+cascade there would compound the divergence); the lane never + promoting a diverged landing; the remedy being cockroach recovery only. +- **Breaks:** silent permanent scheduler/sequencer divergence — the + theft-equivalent failure the whole review centered on (F1/F3 residuals). +- **Anchor-aware frontier (PR5, 2026-06-26):** the content-identity check fires + only at/above the batch-tree **anchor** ([I16](#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). + `populate_safe_accepted_batches` seeds its initial expected nonce from the + anchor (0 for genesis — unchanged; `N'` for a cockroach-recovered deployment), + so L1 landings *below* `N'` are skipped by nonce-mismatch — they are **trusted + collapsed history**, folded into the recovered checkpoint `S'`, not foreign. + This only affects the empty-frontier seed; a running sequencer (non-empty, + append-only `safe_accepted_batches`) always resumes from `latest_accepted`, so + its foreign/zombie detection is byte-identical. `setup --recovery` itself + *defers* frontier population entirely (`InputReader::set_frontier_mode(DeferUntilAnchorSet)`): + its syncs run against an empty tree, so a frontier built then would falsely + diverge — `run`'s first sync populates it once the anchor is set. + +### I16. The batch tree has exactly one valid parentless root, carrying the deployment's anchor nonce + +- **Holds:** since PR5 (cockroach recovery, 2026-06-25). Every batch's nonce is + `parent.nonce + 1`, except the single parentless root, which carries the + `batch_tree_anchor` nonce — `0` for a genesis deployment, `N'` for a + cockroach-recovered one (`setup --recovery` writes the anchor before the + `setup_complete` marker). `run`'s first tip *is* that root (there is no + separate sentinel batch). A fully-torn cascade re-roots parentless at the + same anchor via `open_fresh_tip_in_tx`'s `parent = None` path, after + invalidating the old root — so only one *valid* parentless root ever exists, + invalidated ones coexisting. +- **Enforced by:** `trg_enforce_nonce_contiguity` — its parentless arm is an + *exact* match `nonce == (SELECT nonce FROM batch_tree_anchor)` (tighter than + the pre-PR5 "must be 0"), plus an at-most-one-valid-parentless-root guard + scoped to `invalidated_at_ms IS NULL`; `compute_next_nonce(None)` reads the + same anchor; `trg_batch_tree_anchor_write_once` freezes the anchor once + `setup_complete` exists. Normal (anchor 0) deployments are byte-identical to + pre-PR5. +- **Depended on by:** the submitter resuming at the right nonce — `run` submits + `valid_closed_batches` with `nonce >= frontier_nonce`, where `frontier_nonce` + defaults to the anchor (`= N'`) while `safe_accepted_batches` is still empty + after recovery, so the submitter starts at `N'` rather than 0; the recovery + fill roots the rebuilt tree at `N'` without replaying history. (`N'` is trusted + checkpoint metadata, not re-verified at setup — see + [`docs/recovery/cockroach.md`](recovery/cockroach.md#data-dictionary).) +- **Breaks:** a tree mis-anchored at the wrong nonce ⇒ `run`'s first batch + carries a nonce the scheduler rejects ⇒ the sequencer is wedged (never + submits), or — worse, if defenses were absent — a recovered tree silently + diverging from canonical L1 state. diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md new file mode 100644 index 0000000..2ff8a18 --- /dev/null +++ b/docs/protocol/application-contract.md @@ -0,0 +1,161 @@ +# The Application contract + +The FFI seam. An app plugs into the sequencer by implementing +[`Application`](../../sequencer-core/src/application/mod.rs). The sequencer +assumes the contracts below **without runtime enforcement** — it links the app, +calls it on the hot path and during catch-up, and trusts it to be a pure +deterministic state machine. A violation is not caught; it surfaces as +scheduler/sequencer divergence, which under rollup semantics is +theft-equivalent ([threat model](../threat-model/README.md), "Self-trust"). + +This document **owns** the contract. [`AGENTS.md`](../../AGENTS.md) §"Application +Trait Contract" is the map. The placeholder wallet +([`examples/app-core/`](../../examples/app-core/)) is the reference impl; a +production app will wrap a Cartesi Machine behind the same trait. + +--- + +## The execution methods + +| Method | Mutates? | Clock advance | Failure contract | +|---|---|---|---| +| `validate_user_op(sender, op, current_fee) -> Result<(), InvalidReason>` | **No** — pure, read-only | — | `Err(InvalidReason)` ⇒ op skipped, no state change | +| `execute_valid_user_op(valid, safe_block) -> Result` | Yes | `clock = max(clock, safe_block)` | `Internal` is **fatal** (see *Replay safety*) | +| `execute_direct_input(input) -> Result` | Yes | `clock = max(clock, input.block_number)` | `Internal` is **fatal** | + +`MAX_METHOD_PAYLOAD_BYTES` is the app's declared upper bound on a method +payload's encoded size (selector + args). The sequencer treats it as a sizing +input, not a gate: it derives the per-op byte cost +(`SignedUserOp::max_batch_metadata() + A::MAX_METHOD_PAYLOAD_BYTES`, +[`inclusion_lane/mod.rs`](../../sequencer/src/ingress/inclusion_lane/mod.rs)) to +compute how many ops fit a batch's byte budget. The app must not emit a method +payload larger than it declares. + +### One execution entry point + +User ops are **never** executed by calling `execute_valid_user_op` directly. +They go through the free function +[`validate_and_execute_user_op`](../../sequencer-core/src/application/mod.rs), +which enforces the protocol guard `max_fee ≥ current_fee` *before* app +validation, then calls `validate_user_op`, then `execute_valid_user_op`. It is a +free function — not an overridable trait method — precisely so no impl can skip +the guard. Both consumers (the inclusion lane and the canonical scheduler) call +it; that shared call path is half of the +[duality](scheduler-semantics.md#the-three-implementations-and-why-they-agree) +agreement. + +Consequently `validate_user_op` must **not** re-implement the `max_fee` guard as +its contract (the free function already owns it); it checks only app-level +predicates — nonce match for user-op replay protection, and fee-balance +coverage. (An impl *may* additionally check `max_fee`, as the placeholder does, +but must not *rely* on being the only guard.) + +--- + +## Cross-cutting contracts + +### 1. Determinism & purity + +Execution must be a pure function of `(input, current state)`: + +- **No** `SystemTime::now()`, `HashMap`/`HashSet` iteration order, floating + point, threads, or any other nondeterminism in a consensus path. +- `validate_user_op` is **pure and read-only** — no mutation, no time + dependence, no randomness. State changes flow *exclusively* through the two + execute methods. Mutating from `validate_user_op` breaks replay (validation + runs on a different schedule than execution). +- The same bytes against the same state must always produce the same outcome and + the same `AppOutputs`. This is what lets the off-chain mirror predict the + canonical fold and what lets recovery's `fold_replay` reconstruct state from + L1 alone. + +### 2. Replay safety — `Invalid` vs `Internal` + +The sequencer persists every executed input and, on restart, replays them in +order against a fresh instance to rebuild state (catch-up). Therefore: + +- **Any input that executed successfully live must execute successfully on + replay.** Catch-up treats `AppError::Internal` as **fatal** — it aborts + startup and the sequencer cannot resume. Never return `Internal` for a byte + sequence that previously succeeded. +- Prefer `ExecutionOutcome::Invalid` for malformed or ill-typed input caught at + the app level — `Invalid` is replay-safe (it deterministically skips, live and + on replay). Reserve `AppError::Internal` for genuine invariant violations + ("validated user op cannot pay fee") — real bugs, deliberately fatal, not + adversarial inputs. + +### 3. The safe-block clock — `last_executed_safe_block` + +`last_executed_safe_block() -> u64` returns the **maximum block carried by any +input this instance has executed** (frame `safe_block` for user ops, L1 +`inclusion_block` for directs), or 0 if nothing has executed. + +- It is **carried in execution, not a setter** — every execute method advances + it via `max`, so an app cannot execute an input and forget to move the clock. +- It **must survive `create_dump`/`from_dump` round-trips** (it is part of the + logical state a dump captures). +- Recovery reads it as `A`, the safe block a checkpoint state reflects, and the + `(A, B]` fridge range is reconstructed from it + ([cockroach recovery](../recovery/cockroach.md)). A wrong clock mis-defines + that range. + +`executed_input_count() -> u64` is a diagnostic seam — replay/catch-up and the +snapshot byte-comparison compare a live instance against a replayed one with it. + +### 4. Dump lifecycle round-trip + +The snapshot lifecycle ([snapshots](../snapshots/lifecycle.md)) drives four +dump methods. `Self`-typed methods load/construct; the associated functions are +pure over the path: + +- `create_dump(prefix)` — `prefix` must not already exist; the impl creates and + populates it. **Durability:** on `Ok`, the dump must survive an immediate + kernel crash — the impl must `fsync` the dump's files *and* the directory + entries referencing them (on POSIX: the prefix dir and its parent) **before + returning**. The sequencer inserts the DB row pointing at this path *after* + `create_dump` returns; without the in-method fsync, a crash can leave a + WAL-flushed row pointing at absent bytes. +- `from_dump(prefix)` — rehydrate **equivalent logical state** from a dump this + same impl wrote. Loading another impl's dump is undefined. `create_dump` then + `from_dump` must round-trip: equal logical state, equal + `last_executed_safe_block`, equal `executed_input_count`. +- `state_file_in_dump(prefix)` — a **pure function of `prefix`** (callable + without loading the dump or instantiating the app), returning a single file + (not a directory) whose bytes match what an independent canonical machine's + `inspect_state` would produce for the same logical state. For impls whose + persistence representation already *is* the canonical state, this can be the + same file `create_dump` wrote ([format](../snapshots/format.md)). +- `delete_dump(prefix)` — remove a previously-created dump (GC of superseded + snapshots). + +Genesis construction is intentionally **not** on the trait — it varies per impl +(CLI config for the toy wallet, a machine-image path for a CM-wrapping app) and +lives on the concrete type, called by the runtime at bootstrap. + +--- + +## Who depends on each contract + +| Contract | Depended on by | +|---|---| +| Purity / determinism | the [duality](scheduler-semantics.md) (off-chain prediction = canonical fold); recovery `fold_replay` | +| Replay safety (`Internal` fatal) | catch-up on every restart | +| Safe-block clock survives dumps | cockroach recovery's `A`; snapshot offset accounting | +| `create_dump` in-method fsync | crash-safety of the dump/row ordering ([I13](../invariants.md)) | +| `state_file_in_dump` = canonical bytes | the watchdog / indexers reading finalized state | +| One execution entry point | the `max_fee` protocol guard's non-bypassability | + +## Rejection semantics every app implements + +`InvalidReason` is the protocol's rejection vocabulary — these produce **no state +mutation and are not persisted**: + +- `InvalidNonce` — user-op replay protection. +- `InvalidMaxFee` — the op won't pay the current frame fee (log-space exponent, + base 129/128; see [`fee`](../../sequencer-core/src/fee.rs)). +- `InsufficientFeeBalance` — the sender cannot cover the fee. "Fee", not "gas": + it tracks DA, not compute. + +Deposits are **direct-input-only** (L1 → L2) and must never be represented as +user ops; that is why `execute_direct_input` is required (no default) — a no-op +default would silently strand every deposit. diff --git a/docs/protocol/scheduler-semantics.md b/docs/protocol/scheduler-semantics.md new file mode 100644 index 0000000..f9a31ee --- /dev/null +++ b/docs/protocol/scheduler-semantics.md @@ -0,0 +1,193 @@ +# Scheduler acceptance semantics + +The canonical ordering rule. The on-chain scheduler reads a stream of L1 safe +inputs and folds them into application state in one deterministic order. Every +other component that *predicts* that order — the gold frontier, the inclusion +lane, the recovery fold — must reproduce this algorithm exactly. Disagreement is +silent scheduler/sequencer divergence, the most severe failure in the system +([I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations)). + +This document **owns** that algorithm. [`AGENTS.md`](../../AGENTS.md) §"Sequencer / +Scheduler Duality" is the map; this is the detail. The reference implementation +is [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs) — the same source +compiled into the on-chain canonical machine and run bare-metal by the recovery +fold, so the two targets agree *by construction*. + +--- + +## The one stream, the one classification + +The scheduler consumes `InputAdded` events from the InputBox in L1 order. Each +event carries an authenticated `msg_sender`. Classification is **by sender +address, never by a tag byte** ([`process_input`](../../sequencer-core/src/scheduler/mod.rs)): + +| Sender | Treated as | Effect | +|---|---|---| +| `== sequencer_address` | a **batch** (SSZ-encoded `Batch`) | folded through the acceptance algorithm below | +| anything else | a **direct input** (deposit) | appended to the *fridge* (the direct-input FIFO), drained later | + +The payload is opaque to classification; app-specific decoding happens inside +`Application::execute_direct_input` (see the +[application contract](application-contract.md)). + +--- + +## The algorithm (`process_input`) + +Every input — batch or direct — first runs the **censorship backstop**, then is +classified. + +``` +process_input(input): + 1. force_execute_overdue(input.inclusion_block) ── backstop, runs for EVERY input + 2. if input.sender != sequencer_address: + fridge.push_back(input) ── DirectEnqueued + else: + process_batch_payload(input) ── the acceptance algorithm +``` + +### Step 1 — Force-execute overdue directs (the backstop) + +Before *anything* else, drain every fridge direct that has gone **overdue**: +`current_block − direct.inclusion_block ≥ MAX_WAIT_BLOCKS` +([`force_execute_overdue`](../../sequencer-core/src/scheduler/mod.rs)). This is +the censorship-resistance guarantee: a direct cannot be held in the fridge +forever by a sequencer that freezes `safe_block` or stops submitting batches — +once it ages out, it executes regardless of any frame. It runs on every input +tick (even a malformed batch advances `current_block` and can trip the +backstop). Drained directs execute in FIFO (ascending `inclusion_block`) order. + +### Step 2 — Batch acceptance (`process_batch_payload`) + +A batch from the sequencer address runs this gauntlet, in order. The first +failing gate decides the outcome; only `BatchExecuted` (including the empty-batch +case) **consumes the nonce**. + +| # | Gate | On failure | Nonce? | +|---|---|---|---| +| a | SSZ-decode the payload as `Batch` | `BatchRejected(DecodeFailed)` | not consumed | +| b | `batch.nonce == next_expected_batch_nonce` | `BatchRejected(WrongNonce)` | not consumed | +| c | *empty frames?* → accept as a no-op | `BatchExecuted` | **consumed** | +| d | structural frame check (below) | `BatchRejected(SafeBlockAboveInclusionBlock \| NonMonotonicSafeBlocks)` | not consumed | +| e | staleness: `inclusion_block − frames[0].safe_block ≥ MAX_WAIT_BLOCKS` | `BatchSkippedStale` | **not consumed** | +| f | execute every frame (below) | `BatchExecuted` | **consumed** | + +**Gate d — structural frame check** +([`batch_reject_reason_for_block`](../../sequencer-core/src/scheduler/mod.rs)): + +- every frame's `safe_block ≤ inclusion_block` (a frame cannot claim to have + accounted for L1 state it was included before) — else + `SafeBlockAboveInclusionBlock`; +- frame `safe_block`s are **non-decreasing** across the batch — else + `NonMonotonicSafeBlocks`. + +**Gate e — staleness** is measured against the **first** frame only +(`has_elapsed_since`, mirrored off-chain by +[`ProtocolTiming::is_scheduler_stale`](../../sequencer-core/src/protocol.rs)). A +stale batch is a **true no-op in nonce space**: the expected nonce does *not* +advance, so the next batch — including a freshly-rebuilt one reusing the same +nonce after recovery — is what the scheduler accepts next. This is the +mechanism behind cascading invalidation (AGENTS.md §"Cascading invalidation"). + +### Step 2f — Frame execution order (drain-before-ops) + +For each frame, in order ([`process_batch_payload`](../../sequencer-core/src/scheduler/mod.rs)): + +1. **Drain covered directs first**: pop every fridge direct with + `inclusion_block ≤ frame.safe_block` and execute it + ([`drain_directs_safe_at`](../../sequencer-core/src/scheduler/mod.rs)). The + `safe_block` is the sequencer's commitment that it has accounted for all + directs up to that block; the scheduler enforces the drain regardless. +2. **Then execute the frame's user ops**, each through the single entry point + [`validate_and_execute_user_op`](../../sequencer-core/src/application/mod.rs) + with `(frame.fee_price, frame.safe_block)`. A user op is silently skipped + (no state change, no output) when its signature is unrecoverable, when app + `validate_user_op` rejects it, or when the protocol `max_fee ≥ fee_price` + guard fails — the fold is a pure deterministic function and emits no + diagnostics at the library seam. + +This ordering — directs ≤ `S_K` then ops validated on top of them — is exactly +what the inclusion lane writes into frame K's wire content +([I2](../invariants.md#i2-drain-attribution-drained-directs-land-in-the-new-frame)), +which is what keeps the off-chain prediction faithful. + +--- + +## The three implementations (and why they agree) + +I1 names three places this algorithm lives. They are not three rewrites; two are +*derived* from the first. + +| # | Implementation | Role | Source | +|---|---|---|---| +| 1 | **Canonical fold** `Scheduler` | on-chain authority + bare-metal recovery fold | [`scheduler/mod.rs`](../../sequencer-core/src/scheduler/mod.rs) | +| 2 | **Off-chain predicate** `scheduler_accepts` + `advance_expected_batch_nonce` | builds the gold frontier (`safe_accepted_batches`) | [`protocol.rs`](../../sequencer-core/src/protocol.rs) | +| 3 | **Inclusion lane live prediction** | drains + executes ahead of L1 to issue soft confirmations | [`ingress/inclusion_lane/`](../../sequencer/src/ingress/inclusion_lane/) | + +- **#1 is the authority.** It is the literal scheduler source; the recovery + engine [`fold_replay`](../../sequencer-core/src/scheduler/fold.rs) *drives* it + (seeds the fridge, replays the stream), so recovery is consistent with L1 by + construction rather than by a parallel re-implementation. It is not a fourth + copy. +- **#2 is a predicate, not the full fold.** `scheduler_accepts` answers only + "would the scheduler accept this batch into the frontier?" — it checks sender, + decode, staleness, and nonce, and **deliberately omits gate d** (the + structural frame checks). That omission is sound by *self-trust*: the frontier + simulator runs over the sequencer's **own** batch submissions, which are + well-formed by construction; a structurally-malformed batch would be a + self-bug (a fault state to crash on), not an adversarial input to predict. + This document is the cross-reference home for the omission — the asymmetry is + intentional, not a missing check. +- **The expected-nonce fold** is homed once, next to `scheduler_accepts`, as + [`advance_expected_batch_nonce`](../../sequencer-core/src/protocol.rs). The + submitter's `decide_submit_start` consumes it directly. The frontier builder + `populate_safe_accepted_batches` keeps a deliberate inline copy of the same + advance — its loop interleaves the advance with two storage-only side effects + (the R2 content-identity check and the `canonical_divergence` freeze) that + cannot move below the protocol layer; sharing the fold there would force a + callback contract. The duplication is intentional and documented at the call + site. + +**Why the agreement is load-bearing:** the gold frontier (#2) is what +recovery's cascade pivots on and what promotion trusts; the lane (#3) is what +users see as soft confirmations. If any of the three computes a different +accept/reject/order than the canonical fold (#1), the sequencer will have +promised users a future the scheduler will not produce. No mechanism enforces +the agreement — only review and the duality tests. **Change one, check all.** + +--- + +## Invariants this rests on + +- [I1](../invariants.md#i1-scheduler-acceptance-semantics-agree-across-all-implementations) + — the three implementations agree (this document is its prose). +- [I2](../invariants.md#i2-drain-attribution-drained-directs-land-in-the-new-frame) + — drained directs land in the new frame, so the lane's wire content matches + the fold's drain-before-ops order. +- [I3](../invariants.md#i3-frame-safe_blocks-are-non-decreasing-along-the-spine) + — frame `safe_block`s are non-decreasing, the lane-side mirror of gate d's + monotonicity rule. + +## Test-pinned properties + +The duality's load-bearing edge cases each have at least one test +([`scheduler/mod.rs`](../../sequencer-core/src/scheduler/mod.rs) tests, +[`protocol.rs`](../../sequencer-core/src/protocol.rs) tests): + +- **Empty batches are valid no-ops that consume the nonce** — no first frame to + measure staleness against (`empty_batches_are_valid_noops`, + `scheduler_accepts_empty_frames_batch_regardless_of_age`). +- **Stale batches do not consume the nonce** and the next batch reuses it + (`stale_batch_is_skipped_without_consuming_nonce`). +- **Drain uses an inclusive `inclusion_block ≤ safe_block` rule** + (`frame_drain_uses_consistent_inclusive_safe_block_rule`). +- **The backstop drains all overdue directs before the batch** + (`pre_batch_backstop_executes_overdue_directs_before_user_ops`, + `backstop_drains_all_overdue_directs`). +- **Staleness boundary is `≥ MAX_WAIT_BLOCKS`** — accepted at `MAX_WAIT − 1`, + skipped at `MAX_WAIT` (`scheduler_accepts_boundary_just_below_stale`, + `is_scheduler_stale_reports_true_at_and_past_threshold`). +- **Structural rejects don't consume the nonce** + (`non_monotonic_safe_blocks_invalidate_batch`, + `frame_safe_block_above_inclusion_block_invalidates_batch`, + `wrong_batch_nonce_is_rejected_without_consuming_nonce`). diff --git a/docs/recovery/README.md b/docs/recovery/README.md index 34606c8..6114124 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -8,7 +8,7 @@ See `AGENTS.md` "Batch Staleness and Recovery" for quick-reference tables and fu The sequencer's recovery loop spans two process lifetimes: -1. **In-process detection.** The `DangerDetector` polls `Storage::check_danger` on a cadence. When any non-`Safe` status fires (`L1ViewStale`, `ClosedBatchInDanger`, `TipInDanger`, or `EstimatedBatchInDanger`), the runtime converts that into `DangerDetectorExit::DangerDetected` under `RunError::Worker` and the process exits with non-zero status. +1. **In-process detection.** The `DangerDetector` polls `Storage::check_danger` on a cadence. When any non-`Safe` status fires (`CanonicalDivergence`, `L1ViewStale`, `ClosedBatchInDanger`, `TipInDanger`, or `EstimatedBatchInDanger`), the runtime converts that into `DangerDetectorExit::DangerDetected` under `RunError::Worker` and the process exits with non-zero status. 2. **External respawn.** An orchestrator (systemd, k8s, …) restarts the process. 3. **Startup dispatch.** The fresh boot runs `run_preemptive_recovery` before any writers come online: sync L1, re-run `check_danger`, then `decide_startup_action` routes to one of `Proceed`, `RecoverTip`, `FlushAndCascade`, or `Refuse`. Recovery actions run their DB mutations as single SQLite transactions; `Proceed` intentionally does no DB writes. @@ -43,7 +43,13 @@ Recovery requires at least one Gold ancestor (the cascade invalidates a suffix a The TLA+ model handles this with a **genesis sentinel**: the initial state starts with a Gold batch at nonce 0. This is a modeling technique that eliminates the nonce-0 special case, allowing Resolve to use uniform logic (the `fng > 1` guard is always satisfied). Without it, the model would need a separate Resolve action with different arithmetic for the "no Gold ancestor" case. -The implementation can handle the nonce-0 case either by submitting a sentinel batch at first startup, or by special-casing the recovery code for the "no Gold ancestor" branch. +The implementation handles the nonce-0 case **structurally**: `open_fresh_tip_in_tx` (`storage/ingress.rs`) roots a nonce-0 batch whenever the valid path is empty (genesis, or a fully-torn cascade) — no sentinel batch is submitted and no recovery branch is special-cased. The model's sentinel and the implementation's structural root play the same role. + +#### Cockroach recovery generalizes the root nonce (the anchor) + +Cockroach recovery (`setup --recovery`) rebuilds a wiped DB from a trusted checkpoint and must resume submitting at nonce `N'` without replaying history — so the rebuilt tree is rooted at `N'`, not 0. Rather than plant a fake "sentinel" batch at `N'-1`, PR5 generalizes the structural root: a `batch_tree_anchor` singleton holds the nonce the parentless root carries (default `0`; recovery sets `N'`). The same `open_fresh_tip_in_tx` / `compute_next_nonce(parent = None)` path then roots `run`'s first tip at `N'`, and `trg_enforce_nonce_contiguity` validates the root against the anchor (exact match) instead of a hard-coded 0. There is **no sentinel batch row** — the root tip *is* the anchored batch. Normal deployments keep anchor `0` and are byte-identical. See [I16](../invariants.md) and the [cockroach-recovery design](#cockroach-recovery-setup---recovery) below. + +A sealed `N'-1` sentinel was considered and rejected: a valid closed batch at `N'-1` is a legal cascade pivot, so a runtime cascade could invalidate it and leave the tree re-rooting at 0 (ABORTed by the unchanged contiguity trigger) — an unguarded reliance on "the frontier never drops to `N'-1`". The anchor has no such hidden dependency. ## Coloring @@ -192,9 +198,9 @@ Stop accepting new user operations. From the outside world, the sequencer is tem ### Step 3: Flush mempool -Query the latest confirmed `w_nonce` (N) and the pending `w_nonce` (M). Submit `M - N` no-op transactions (e.g., self-transfer of 0 ETH) at nonces N, N+1, ..., M-1. These compete with any batches in the mempool at the same slots. +Read the persisted **wallet-nonce watermark** `W` — the highest `w_nonce` this deployment ever broadcast (`wallet_nonce_watermark` singleton; see Implementation Constraint 1). Query the latest confirmed `w_nonce` (N) and the pending `w_nonce` (M). Submit no-op transactions (self-transfers of 0 ETH) at nonces N, N+1, ..., `max(M, W+1) - 1`. These compete with any of our transactions still alive anywhere in the network — including zombies the local node's pool has forgotten (review F1). -Wait for all `M - N` slots to reach L1 safe finality. +Wait until both `pending <= safe` **and** `safe >= W + 1`: every slot this deployment ever used is consumed at safe depth. The second conjunct is the durable anchor — without it the flush trusts the local node's volatile mempool memory, which a dropped-locally-but-alive-elsewhere zombie evades entirely. The flush reports the safe block at which it observed resolution; Step 5 refuses to cascade until the re-synced view reaches at least that block (review F2). ### Step 4: Post-flush state @@ -205,7 +211,7 @@ Every `w_nonce` slot from N to M-1 is now resolved: There are no more mempool entries. All uncertainty is resolved. -**Flush safety does not depend on eviction.** A no-op may fail to evict a still-pending batch tx (e.g. our local node rejects the replacement under EIP-1559's ≥10% bump rule). That's fine: the outer `flush_and_wait` loop is unbounded — it keeps running until `pending ≤ safe`, and *eventual* inclusion of either the original batch tx or the no-op resolves the slot. Safety holds regardless of which lands; eviction is only an operational efficiency concern. +**Flush safety does not depend on eviction.** A no-op may fail to evict a still-pending batch tx (e.g. our local node rejects the replacement under EIP-1559's ≥10% bump rule). That's fine: a rejected send surfaces as a hard `FlushError` and the process exits, the orchestrator respawn re-runs the flush, and *eventual* inclusion of either the original batch tx or the no-op resolves the slot — the unbounded retry lives in the respawn loop, not inside `flush_and_wait`. Safety holds regardless of which lands; eviction is only an operational efficiency concern. ### Step 5: Run recovery @@ -225,7 +231,9 @@ Any batch past that gold frontier is **doomed**, in one of three concrete senses **Why isn't this just "stale"?** Under self-trust (we don't defend against malformed self-submissions), the *first* non-gold closed batch can only be Silver-stale or Pending. Nonce-mismatch is impossible at the frontier — nonces are contiguous on the valid path (`trg_enforce_nonce_contiguity`). But *downstream* batches past that first non-gold are typically Silver-fresh-poisoned: their inclusion-staleness was fine, but they were processed when expected was stuck at the poisoned nonce. -Cascading from the first non-gold catches all three. **No per-batch age check is needed for the cascade pivot itself** — every closed batch past gold is doomed by construction. +A **fourth shape** sits outside this taxonomy: a closed batch that was **never submitted** (closed after the submitter's last tick before the detector exit). It has no L1 footprint, no killed tx, and is not literally doomed — it could simply be submitted after recovery. The cascade invalidates it anyway: once committed to recovery, cascading the entire non-gold suffix converges in one cycle and avoids spine-order reasoning about a half-submitted suffix. The cost is real (its soft confirmations are rolled back); this is a deliberate convergence-over-preservation policy choice. + +Cascading from the first non-gold catches all four. **No per-batch age check is needed for the cascade pivot itself** — every closed batch past gold is either doomed by construction or sacrificed by the convergence policy. #### Path A — `recover_post_flush(danger_threshold)` (called from FlushAndCascade) @@ -337,11 +345,47 @@ A killed batch acts as **silent nonce poison**: the scheduler never sees it, so Dead batches occupy `w_nonce` slots strictly below `walletNonce`. Recovery batches occupy `w_nonce` slots at or above `walletNonce`. **No overlap.** This is why no mutual exclusion is needed between dead batches and recovery batches -- they live in non-overlapping `w_nonce` ranges. +## Cockroach recovery (`setup --recovery`) + +Everything above is **standard recovery**: the running sequencer's own bookkeeping (the batch tree, pending dumps) lets it cascade a doomed suffix and resume. It is automatic and in-process. + +**Cockroach recovery** is the catastrophe path — the local DB is lost or has diverged (`CanonicalDivergence`, [I15](../invariants.md)). There is no tree to cascade; the operator wipes the DB and rebuilds canonical logical state from a trusted checkpoint plus L1. It is an operator-driven, one-shot `setup` mode, not a runtime action. The summary: + +Given a trusted checkpoint machine `S` at block `B` (a finalized `dumps//` dir, carrying `N` = its resume nonce and `A` = its last-executed safe block), `setup --recovery --checkpoint-block B --checkpoint-dump-dir ` runs **flush → fold → fill**: + +1. **Flush** the wallet nonce (keyed — recovery, unlike plain `setup`, signs) so every previous-instance batch resolves at safe depth `≤ C`, the post-flush safe head. Re-sync `safe_inputs` through `C`. +2. **Fold** (the pure `sequencer-core` engine, shared with the on-chain scheduler so it is consistent by construction): seed the fridge from the `(A, B]` directs (drop batches — already in `S`), replay the `(B, C]` stream, drain the leftover fridge at `C`. Yields `(S', N')` = the advanced app state and the resume nonce. +3. **Fill** a consistent DB: snapshot `S'` as finalized at `C`; **anchor the batch tree at `N'`** ([I16](../invariants.md) — the root tip *is* `N'`, no sentinel batch); sequence the `≤ C` directs so the replay cursor starts past them (they're already in `S'`, while `run`'s first on-chain batch re-drains them by `safe_block`). `run` boots from this state. + +During recovery the gold frontier (`safe_accepted_batches`) population is **deferred** (`FrontierMode::DeferUntilAnchorSet`): the tree is empty until fill, so simulating acceptance against it would flag every L1 batch as foreign and freeze the frontier ([I15](../invariants.md)). It is populated on `run`'s first sync — once the anchor `N'` is set — so the folded `< N'` history is skipped as trusted collapsed history. `N` is **trusted checkpoint metadata**, not re-verified at recovery time: a wrong-low `N` surfaces at `run` via the content-identity check, but a wrong-high `N` does not — sound because a sequencer-produced finalized dump cannot carry a wrong `N` by construction (see [`cockroach.md`](cockroach.md#data-dictionary) for the full trust boundary). Recovery is a **strict one-shot**: it refuses (terminal) on a DB that is already set up — the model is "wipe and re-run", and a crash before the `setup_complete` marker re-runs cleanly (the fill is idempotent). + +The detect-and-refuse gate is the *trigger*: a fresh `setup` that finds a previous instance's batches past the checkpoint refuses with exit `40` (`EXIT_SETUP_NEEDS_RECOVERY`), pointing the operator here. + +## Canonical divergence (terminal, outranks every arm) + +Independent of the staleness machinery, the input reader's acceptance +simulation cross-checks every **accepted** landing against the local batch at +that nonce (content-identity check, review R2: `keccak256` of the landed wire +bytes vs the hash stamped at seal). On mismatch — or on an accepted landing +with no valid closed local batch at all — it persists the +`canonical_divergence` marker atomically with the sync, freezes the +acceptance frontier, and `check_danger` reports `CanonicalDivergence` +**ahead of every other arm**, so a respawn loop can never route a diverged +node into `Proceed` or `FlushAndCascade`. The startup dispatch maps it to a +terminal `Refuse`. + +The remedy is **cockroach recovery (wipe + rebuild from L1), never the +standard recovery on this page**: the cascade reconciles the batch tree's +*shape* under the assumption that accepted nonce N is our batch N — a content +mismatch means canonical state contains executed effects with no reliable +local source, so rebuild-from-L1 is the only honest repair. + ## Implementation Constraints These constraints were discovered during TLA+ model checking and are required for correctness: 1. **`walletNonce` must NOT be reset during recovery.** Recovery batches must use `w_nonces` strictly past all dead batch slots. The flush consumes dead batch slots by advancing `nextL1Slot` up to `walletNonce`. Recovery starts fresh from there. + **Mechanism (review R1a):** `walletNonce` is realized durably as the `wallet_nonce_watermark` singleton — the highest wallet nonce ever broadcast. Every broadcaster (the batch poster and the flusher's no-ops alike) commits `watermark = max(watermark, n)` power-loss-durably (`synchronous=FULL`) **before** sending at nonce `n` (write-before-broadcast; a crash between commit and send only over-covers — one wasted no-op). The flush's completion condition is `pending <= safe && safe >= watermark + 1`, so it cannot declare victory while any slot we ever used is unresolved — restoring this constraint against the local pool's volatile memory (review F1). The watermark is never reset and never lowered. 2. **`SubmitBatch` must use `max(walletNonce, nextL1Slot)`.** Prevents assigning `w_nonce` values for slots L1 has already consumed. @@ -361,7 +405,7 @@ The recovery design is verified with bounded TLA+ model checking. The canonical Models the core slot-level mechanics of preemptive recovery. At every `w_nonce` slot, L1 non-deterministically includes the spine batch OR a flush no-op (killing the batch). This covers the case where the frontier batch itself is killed during flush. The model also treats the open Tip's `safe_block` as meaningful, so it can explicitly recover an aging Tip that has no L1 footprint yet. -The model is a **safety over-approximation**: it allows `AdvanceTip` and `SubmitBatch` to interleave freely with recovery, which the real protocol prevents (the sequencer goes offline). This makes the proof stronger -- if `ZombieSafety` holds under more interleavings, it holds under fewer. However, the model does not verify the full sequential protocol phases (cutover, flush, wait, recover, resume) described above; in particular, the startup decision of whether a closed unresolved batch must flush before recovery remains an external argument layered on top of the slot-level proof. +The model is a **safety over-approximation for the actions it shares with the implementation**: it allows `AdvanceTip` and `SubmitBatch` to interleave freely with recovery, which the real protocol prevents (the sequencer goes offline). This makes the proof stronger -- if `ZombieSafety` holds under more interleavings, it holds under fewer. However, the over-approximation claim does **not** hold action-for-action — two implementation actions sit *outside* the model's transition set: (1) the model discards an aging Tip only at `MAX_WAIT_BLOCKS`, while the implementation invalidates at `danger_threshold` (= `MAX_WAIT − MARGIN`); (2) the model's `Resolve` has no case for a killed-Pending frontier (it relies on resubmission until the frontier is Silver), while `recover_post_flush` cascades killed Pendings unconditionally. For those actions the implementation's enabled transitions are a *superset* of the model's, so TLC has not explored them; their safety rests on external arguments (an open Tip and a killed Pending have no L1/scheduler state to disagree with). The model also does not verify the full sequential protocol phases (cutover, flush, wait, recover, resume) described above; in particular, the startup decision of whether a closed unresolved batch must flush before recovery remains an external argument layered on top of the slot-level proof. **Verified**: 157M states, 0 violations. diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md new file mode 100644 index 0000000..6bc6b6f --- /dev/null +++ b/docs/recovery/cockroach.md @@ -0,0 +1,196 @@ +# Cockroach recovery (`setup --recovery`) + +The catastrophe path. When the local DB is lost or has diverged +([`CanonicalDivergence`](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)), +there is no batch tree to cascade — the operator **wipes the data dir and +rebuilds canonical logical state from a trusted checkpoint plus L1**. It is an +operator-driven, one-shot `setup` mode, not a runtime action. + +Contrast with **[standard / preemptive recovery](README.md)** (the rest of +`docs/recovery/`): that runs *inside* a live sequencer, uses its own batch tree +to cascade a doomed suffix, and shares the flush machinery. Cockroach recovery +discards the tree entirely and reconstructs `(S', N')` by *folding* L1. + +The pure fold engine ([`sequencer-core/src/scheduler/fold.rs`](../../sequencer-core/src/scheduler/fold.rs)) +is the same scheduler source compiled into the on-chain canonical machine — so +the reconstruction is consistent with L1 *by construction*, not by a parallel +re-implementation. + +--- + +## Data dictionary + +Five quantities drive the procedure. Knowing where each is *born* is the key to +reading the code. + +| Symbol | Meaning | Where it comes from | +|---|---|---| +| **`S`** | The trusted checkpoint machine/app state at block `B`. | Loaded from the dump: `A::from_dump(checkpoint_dump_dir)`. | +| **`A`** | `S`'s last-executed safe block. The fridge is reconstructed from directs in `(A, B]`. | App query: `S.last_executed_safe_block()`. (Persisted in the dump — see `docs/snapshots/format.md`.) | +| **`B`** | The checkpoint's L1 inclusion block. `S` reflects every **batch** with inclusion `≤ B` and **no direct** in `(A, B]`. | Operator arg: `--checkpoint-block`. | +| **`N`** | The checkpoint's resume batch nonce (the scheduler counter at `B`). The bare-metal app *cannot* recompute it, so it rides as checkpoint metadata. | `info.toml`'s `next_batch_nonce` in the dump. | +| **`C`** | The post-flush safe head — the stopping block. The flush guarantees every previous-instance batch is settled at safe depth `≤ C`. | Return of `flusher.flush_and_wait(...)`. | +| **`N'`** | The resume nonce the new sequencer submits at — `N` advanced by the accepted batches in `(B, C]`. Becomes the **batch-tree anchor**. | Output of `fold_replay(...)`. | + +**Invariant `A < B`** (checked at load): the checkpoint's executed state must +predate its inclusion block, or the `(A, B]` fridge range is ill-defined. + +`N` and the `(A, B]` batch content are **operator-trusted inputs** — there is no +cheap on-chain oracle to validate the checkpoint against, and recovery +does **not** re-verify them. The only sound independent check would replay the +scheduler's nonce fold from genesis through `C` (a from-genesis frontier is the +one quantity independent of the trusted `N`) — i.e. re-fetch and reprocess all of +L1, which recovery deliberately does not do. + +This is sound because the checkpoint is a **sequencer-produced finalized dump**, +where the tuple is self-consistent by construction: `info.toml`'s +`next_batch_nonce` is the closing batch's nonce + 1, and a dump only reaches +*finalized* once that batch is promoted (observed accepted on L1), so `N` is +exactly the scheduler's position at `B`. A wrong `N` can therefore arise only +from a corrupted or externally-produced checkpoint — already outside the trust +boundary, the same one that accepts `S` with no verifier. + +If that boundary is ever violated, the two wrong-`N` shapes are **not** +symmetric: +- a wrong-**low** `N` (or a mis-stated `(A, B]`) surfaces loudly at `run` — the + rebuilt root collides with the still-live L1 batches in `[N', M)` and trips the + content-identity check (I15); +- a wrong-**high** `N` is **not** caught — the `[M, N)` history was never reached + on-chain, so nothing collides; the off-chain frontier accepts the local batch + against itself while the real scheduler ignores it. This is why the checkpoint + must be a trustworthy finalized dump, not merely "some app bytes". + +--- + +## The procedure: flush → fold → fill + +``` + ┌─ load S, derive A & N, require A < B + checkpoint │ + (S @ B, N)│ flush wallet nonce ───────────────► C (post-flush safe head) + │ │ + │ ▼ + L1 ────────┼──► re-sync safe_inputs through C (frontier population OFF) + │ │ + │ ▼ + │ source seeds = (A,B] directs (drop batches: in S) + │ replay = (B,C] stream + │ │ + │ ▼ + │ fold_replay(S, N, seeds, replay, C) ──► (S', N') + │ │ + ▼ ▼ + fill: anchor = N', root tip @ N', finalized snapshot of S', + replay cursor past the ≤C directs ──► run boots here +``` + +1. **Load `S`; derive `A`, `N`; require `A < B`.** Read the dump + (`from_dump` + `info.toml`); `A = S.last_executed_safe_block()`. +2. **Flush → `C`.** Settle the wallet nonce (keyed L1 no-ops; this is where + cockroach recovery composes with the standard flush). On completion every + previous-instance batch is resolved at safe depth `≤ C`. `C` is the stopping + point: directs beyond `C` are `run`'s job, not the fold's. +3. **Re-sync `safe_inputs`; F2 coherence.** The reader syncs to the *live* safe + head `H1` (normally `> C` — real time passed while the flush awaited safe + finality); refuse only if it *lags* `C` (a load-balanced RPC replica could + serve a stale view). So `safe_inputs` ends up holding directs through `H1`, + not just `C` — step 6 is careful to drain only the `≤ C` ones. **Gold + frontier population is OFF for recovery's syncs** — the tree is empty until + step 6, so populating the frontier would mark every L1 batch `Foreign` and + falsely freeze it. The frontier is deferred to `run`'s first sync. +4. **Source the fold inputs.** Seeds = the `(A, B]` **directs** (drop + `sender == batch_submitter` — those are batches, already in `S`); replay = + the full `(B, C]` stream. The seed filter *is* the scheduler's own + sender-based classification. +5. **Fold `(S, N)` → `(S', N')`.** The engine seeds the fridge from `(A, B]`, + replays `(B, C]` (force-executing overdue directs, applying accepted + batches, draining covered fridge directs), drains the leftover fridge at `C`, + and advances the nonce to `N'`. +6. **Fill the DB.** Snapshot `S'` as finalized at `C`; **anchor the batch tree + at `N'`** (the root tip *is* `N'` — there is no sentinel batch); open the root + tip at frame `safe_block = C` and sequence **only the `≤ C` safe inputs** so + the replay cursor starts past them. The drain is sender-unfiltered — it + includes the `≤ C` batch-submitter rows alongside the user directs the fold + folded into `S'`, exactly as the genesis tip drains its whole span; those + rows are sequenced (cursor padding), never executed, so the + `sender != batch_submitter` *seed* filter does not reappear here. The `(C, H1]` + directs the resync pulled in past `C` stay **undrained** — `run`'s lane leads + and executes them exactly once as the safe frontier advances `C → H1`. + (Draining them here instead would skip them on catch-up while `S'` never + executed them — a vanished deposit / divergence; this is why the fill uses a + `≤ C`-capped `open_recovery_tip`, not the generic whole-table drain.) `run` + boots from this state. + +--- + +## Code map + +| Step | Code | +|---|---| +| entry / branch | [`setup.rs` `setup()`](../../sequencer/src/runtime/setup.rs) branches on `config.recovery` after the shared prefix (identity pin + initial sync) | +| 1. load + `A < B` | [`recover()` step 1](../../sequencer/src/runtime/setup.rs) — `from_dump`, `read_info`, `CheckpointNotBeforeBlock` | +| 2. flush → `C` | `recover()` step 2 — `MempoolFlusher::flush_and_wait` (see [`runtime/flush.rs`](../../sequencer/src/runtime/flush.rs)) | +| 3. re-sync + coherence | `recover()` step 3 — `set_frontier_mode(DeferUntilAnchorSet)` + `sync_to_current_safe_head` + `ResyncBehindFlushView` | +| 4. source seeds/replay | `recover()` step 4 — `Storage::safe_inputs_in_block_range` + the `sender != submitter` filter + `to_fold_input` | +| 5. fold | [`fold_replay`](../../sequencer-core/src/scheduler/fold.rs) | +| 6. fill | [`fill_recovery_state`](../../sequencer/src/runtime/setup_fill.rs) — anchor + `open_recovery_tip` (`≤ C`-capped drain at frame `safe_block = C`) + finalized snapshot | +| anchor mechanism | [`trg_enforce_nonce_contiguity`](../../sequencer/src/storage/migrations/0001_schema.sql) + `compute_next_nonce` + the anchor-aware frontier in [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) | + +--- + +## Load-bearing constraints & invariants + +- **`A < B`** — checked at load ([`SetupRecoveryError::CheckpointNotBeforeBlock`]). +- **Disjoint fold ranges** — seeds `(A, B]`, replay `(B, C]`, strictly disjoint; + the fold's always-on asserts enforce ascending order *within* each and a + strict block boundary *between* (directs at block `B` are seeds; the replay + starts strictly after). +- **Frontier deferral** — recovery's syncs never populate the gold frontier (the + tree is empty); `run`'s first sync populates it once `anchor = N'` is set, so + the folded `< N'` batches are skipped as trusted collapsed history rather than + flagged `Foreign`. See the anchor-aware-frontier note on + [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). +- **Anchored root** — the rebuilt tree has exactly one valid parentless root, + carrying `N'` ([I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). +- **No double-execution, no lost directs** — the `≤ C` directs are sequenced + (cursor advances) but the finalized snapshot's `l2_tx_index` is set *after* + sequencing, so `run`'s catch-up (`offset > l2_tx_index`) skips them; they are + already in `S'`. Symmetrically, the `(C, H1]` directs are **not** sequenced + here (`open_recovery_tip` caps the drain at `C`), so the cursor sits below them + and `run`'s lane leads + executes them exactly once — neither double-executed + nor lost. + +--- + +## Crash-safety & idempotency + +`setup --recovery` is a **strict one-shot** on a freshly-wiped DB: + +- It **refuses** (terminal, exit 30) if `setup_complete` already exists — the + model is "delete the data dir and re-run", not resume-a-live-deployment. +- The `setup_complete` marker is the linearization point, written **last**. +- A crash *before* the marker is handled fail-loud, not by blind resume: + - A **completed** fill (finalized snapshot present — the last write) re-runs as + a safe no-op; the anchor and root tip are already in place. + - A **same-`N'`** re-run of a fill that crashed *mid-fill* (root tip exists, no + finalized snapshot) is **refused** (`PartialRecoveryIncomplete`). It is *not* + idempotent: a re-sync may have advanced `C` with new directs (which leave + `N'` unchanged) that resuming would leave unsequenced, leaving the snapshot + cursor behind `S'` and double-draining them on `run`. Wipe and re-run. + - A **different-`N'`** re-run (a different checkpoint, or the same one after + `C` advanced with new accepted *batches*) is **refused** + (`PartialRecoveryMismatch`): the durable root tip carries the old nonce and + cannot be silently re-anchored. Wipe and re-run. + - `setup --recovery` over **foreign residue** — a finalized snapshot with no + root tip, left by a crashed plain `setup` (which writes the genesis snapshot + before its marker) — is **refused** (`RecoveryOverResidualSnapshot`). A + completed cockroach fill always has both a snapshot and a root tip; keeping + the old snapshot would mark setup complete over genesis instead of `(S', N')`. + Wipe and re-run. + - A subsequent **plain `setup`** over recovery residue is **refused** + (`GenesisOverRecoveryResidue`, anchor `≠ 0`) — it must not root genesis at + the recovery nonce. + +(See the deep-review remediations: these guards close the partial-recovery +revalidation gap, including the same-`N'`/advanced-`C` double-drain — external +review 2026-06.) diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index 7babc0b..b997cda 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -68,19 +68,18 @@ and its canonical state coincide; one write per `create_dump`. ``` {prefix}/ - state SSZ-encoded WalletSnapshotV1 bytes + state SSZ-encoded WalletSnapshot bytes ``` ## Toy Wallet Wire Format - **Encoding**: SSZ -- **Top-level type**: `WalletSnapshotV1` (Rust struct name; the version is - protocol-external — see below) +- **Top-level type**: `WalletSnapshot` - **Byte order for balances**: big-endian 32-byte integers (`U256`) ### Schema -`WalletSnapshotV1`: +`WalletSnapshot`: - `erc20_portal_address` (`[u8; 20]`) - `supported_erc20_token` (`[u8; 20]`) @@ -92,6 +91,14 @@ and its canonical state coincide; one write per `create_dump`. - `address` (`[u8; 20]`) - `nonce` (`u32`) - `executed_input_count` (`u64`) +- `last_executed_safe_block` (`u64`) — the app's safe-block clock + (`Application::last_executed_safe_block`): max block carried by any + executed input. Recovery reads it as `A`, the safe block this state + reflects, so it must live in the canonical state bytes (both the + bare-metal and canonical-machine sides advance it identically). + +`last_executed_safe_block` was added before any environment was deployed; there +is a single, unversioned schema today (see [Versioning](#versioning)). ### Determinism @@ -122,20 +129,22 @@ comparable. ## Versioning -The struct name carries the wire-format version (`WalletSnapshotV1`), but -the encoded bytes contain no leading version tag. Version dispatch is -expected to live outside the bytes — for example, in an HTTP route prefix -(`/state/v1/...`), a `Content-Type` header, or whatever protocol-level -mechanism the consumer and sequencer agree on. +There is a single, unversioned schema: `WalletSnapshot`. The encoded bytes carry +no leading version tag, and — because there is no backward-compatibility +requirement yet (no long-lived deployment whose dumps a newer binary must read) — +the struct name carries no version suffix either. An earlier draft distinguished +a `V1`/`V2` pair (the `last_executed_safe_block` field was added before any +environment existed); that split was collapsed since no `V1` dumps ever survived. -Future breaking changes must: +If a future change ever needs to break the wire format against live dumps: -1. Introduce a new versioned schema type (e.g. `WalletSnapshotV2`). -2. Provide explicit dispatch at the protocol layer so consumers know - which decoder to use for a given dump. +1. Introduce a new, explicitly versioned schema type (e.g. `WalletSnapshotV2`). +2. Provide explicit dispatch at the protocol layer — an HTTP route prefix + (`/state/v2/...`), a `Content-Type` header, or whatever the consumer and + sequencer agree on — so consumers know which decoder to use; the bytes + themselves stay tag-less. -Do not reorder, repurpose, or reinterpret existing fields in place -without a version bump. +Until then, do not reorder, repurpose, or reinterpret existing fields in place. ## Trust Model diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index ea11a16..27c9261 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -87,16 +87,36 @@ Three SQLite tables back it (`storage/migrations/0001_schema.sql`): | `pending_snapshots` | `(nonce, dump_id, l2_tx_index)` — snapshots of closed-but-not-yet-L1-confirmed batches | | `finalized_snapshot` | single row `(dump_id, inclusion_block, l2_tx_index)` — the latest L1-confirmed state | -`prefix` is an opaque directory the `Application` owns (see [`format.md`](format.md)); -the lifecycle code never looks inside it except through the trait. The split -between **pending** and **finalized** mirrors the sequencer's optimism: a batch -closes off-chain (soft) → its snapshot is *pending*; the batch lands safe on L1 -→ its snapshot is *promoted* to finalized. +`prefix` is the **dump directory** — a structured dir the sequencer owns +(`ingress/inclusion_lane/dump_info.rs`): + +```text +dumps// + state app-owned subtree — the prefix handed to + Application::{create_dump, from_dump, delete_dump} + (opaque to the sequencer; see format.md) + info.toml sequencer-owned checkpoint metadata: + format_version, next_batch_nonce (N), l2_tx_index, + promoted_inclusion_block (B) +``` + +`info.toml` makes a finalized dump a **self-contained checkpoint** for the +recovery handoff: `N` and the replay cursor are known at batch +close and written then; `B` is known at promotion and stamped **in place** +afterwards (and re-stamped from the authoritative DB row at every startup, +closing the commit-then-stamp crash window). An in-place update of a file +*inside* the dir changes no path, so the no-dangling-row invariant, leases, +and GC — all keyed on the immutable directory path — are untouched. The dir +name itself stays opaque; metadata lives only in `info.toml`. + +The split between **pending** and **finalized** mirrors the sequencer's +optimism: a batch closes off-chain (soft) → its snapshot is *pending*; the +batch lands safe on L1 → its snapshot is *promoted* to finalized. The storage half lives in `storage/snapshot_dumps.rs` (SQLite only — no -filesystem); the lane half in `ingress/inclusion_lane/snapshot.rs` (drives the -trait and FS cleanup). That split is load-bearing for the GC crash-ordering -(§7). +filesystem); the lane half in `ingress/inclusion_lane/snapshot.rs` + +`dump_info.rs` (drives the trait and FS work). That split is load-bearing for +the GC crash-ordering (§7). ## 2. The always-load invariant @@ -113,9 +133,13 @@ fail-loud as `CatchUpError::NoSnapshot`, never a branch the happy path handles. When the lane closes a batch (`close_batch_with_snapshot`), ordering is chosen for crash/error safety: -1. **`create_dump` first, outside any transaction.** The `Application` writes - and `fsync`s the dump. On failure nothing is sealed — the batch stays the - open Tip and the lane retries next pass. +1. **The dump directory first, outside any transaction** + (`dump_info::create_dump_dir_with_info`): the dir, its `info.toml` + (`next_batch_nonce` = closing nonce + 1, the replay head; `B` left for + promotion), then the `Application`'s dump under `state` — all written and + `fsync`ed. On failure nothing is sealed — the batch stays the open Tip; the + error propagates per the lane's fail-loud policy (the process exits, and the + retry happens on the next boot after catch-up). 2. **One transaction seals the batch, opens the next, and inserts the `pending_snapshots` row** (`close_frame_and_batch_with_pending_dump`). A committed close therefore *always* has a promotable pending row; a tx failure @@ -334,13 +358,28 @@ in `dumps`; runs *after* (2) so the genesis prefix is registered and not swept). Danger-zone recovery (`storage/recovery.rs`, see [`../recovery/README.md`](../recovery/README.md)) cascade-invalidates batches -that the canonical stream will never reach. When it does (`!invalidated -.is_empty()`), it clears `pending_snapshots` **in the same transaction** as the -cascade — those pendings represent states catch-up must never load. `finalized` -is untouched (its bytes are for an L1-confirmed batch, which survives any -cascade). A **no-op** recovery (closed batches gold, Tip fresh) deliberately -preserves in-flight pendings the lane is still working with. This conditional -clear is why catch-up can safely resume from a surviving pending (§4). +that the canonical stream will never reach. In the same transaction as the +cascade it clears `pending_snapshots` **scoped to the cascade**: only rows +with `nonce >= pivot.nonce` — exactly the cascaded batches' pendings, which +catch-up must never load. (Review F9; implemented in `cascade_and_reopen`, +the shared tail of both recovery paths.) + +Pendings of *gold but not-yet-promoted* batches (landed and accepted while +the process was down) carry lower nonces and **survive**: catch-up resumes +from the freshest surviving checkpoint, and the rows are cleaned up by the +next promotion's `DELETE <= max_nonce`. The scoping makes the §6 +promote-wedge **unrepresentable** rather than unreachable: any nonce the lane +can later observe as accepted either has its pending row intact or belongs +to a post-recovery batch with a fresh row. (The earlier blanket clear was +safe only through a chain of cross-file couplings — same-tx full-backlog +reopen drain, `check_danger` arm ordering, frame-safe-block monotonicity — +documented in the 2026-06-10 review, F9.) In the `RecoverTip` path the +scope deletes nothing: the Tip never has a pending row. + +`finalized` is untouched (its bytes are for an L1-confirmed batch, which +survives any cascade). A **no-op** recovery (closed batches gold, Tip fresh) +deliberately preserves in-flight pendings the lane is still working with. +This is why catch-up can safely resume from a surviving pending (§4). ## 9. Where the code lives diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 8797ecc..8bdf9a1 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -19,7 +19,7 @@ What we are protecting: |-------|-------|--------------| | InputBox contract | Trusted | Authenticates `msg_sender` on `addInput`. Use correctly; do not model forgery. | | Our Ethereum node | Trusted, fail-stop | Inside our infra. May become unreachable; will never lie. | -| Fallback RPC (Infura / Alchemy) | Semi-trusted, fail-stop | Liveness fallback during primary outages. May withhold or delay. Never byzantine. | +| RPC endpoint (`SEQ_ETH_RPC_URL`) | Trusted, fail-stop — **must be one consistent node** | The code supports exactly **one** endpoint, shared by reader, submitter, poster, and flusher; no fallback tier exists yet. Behind a load-balanced fleet, lagging replicas can silently truncate `get_logs` ranges and desynchronize the flush/re-sync views (review F5/F2). The reader now fails loud on an incomplete InputAdded set (F5): a per-app index contiguity check (right prefix) plus a `getNumberOfInputs` count witness pinned at the scanned safe block (complete prefix), so a dropped/clamped/truncated-tail input is detected before the safe head advances rather than silently skipped, and recovery refuses if the re-sync lags the flush view (F2). The residual fleet exposure is a node that lies *consistently* about both its logs and its input count — outside the fail-stop model. A semi-trusted fallback tier (Infura/Alchemy) is future work. | | Operator env / CLI flags | Trusted | Configuration is authoritative. | | Batch-submitter private key | Private | Held in operator infra. Not reachable by the network. | | Sequencer's own code | Trusted (bug-free is a precondition) | Bugs are caught via tests and review, not defended against at runtime. See "self-trust" below. | @@ -30,16 +30,18 @@ What we are protecting: ### Self-trust -The sequencer trusts that its own code is correct. If the sequencer emits a malformed batch, frame, or user op, it is already in a bug state that requires manual intervention — we do not layer runtime defenses against sequencer self-misbehavior. Recovery addresses liveness failures (infrastructure outages, network partitions, gateway failure), not bug-induced malformed state. +The sequencer trusts its own code in a specific sense: **impossible states are never *handled*.** There are no graceful fallback paths, no re-validation of a neighbor module's answer, no code that keeps running past a violated internal contract. If the sequencer emits a malformed batch, frame, or user op, it is in a bug state that requires manual intervention; recovery addresses liveness failures (infrastructure outages, network partitions, gateway failure), not bug-induced malformed state. -This is not an excuse to skip validation at trust boundaries. Inputs from untrusted actors are validated rigorously. Internal invariants are enforced by type system, SQL constraints, and tests — not by defensive runtime checks against hypothetical self-misbehavior. +This is **not** a prohibition on checking. Internal invariants are enforced loudly wherever a check is near-free — the type system, SQL constraints and triggers, boundary assertions — because in this system a loud crash is recoverable by design (orchestrator respawn + startup recovery), while a silently-tolerated bug that externalizes (a signed batch, an ack, a feed event) is state divergence: as severe as theft and undefendable at runtime. The rule, in short: **assert real invariants, fail loud, never absorb silently, never handle gracefully.** The decision test and the register of cross-module invariants live in [`docs/invariants.md`](../invariants.md). + +Inputs from untrusted actors are validated rigorously, as ever. ## In-scope failure modes - L1 provider outages (primary and fallback), minutes to hours - Process crashes at arbitrary points, including mid-transaction - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders -- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. The recovery flusher is load-bearing for this threat: it consumes every pending `w_nonce` slot with a no-op so zombies cannot claim them. +- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, review R1a) so zombies cannot claim them; and the content-identity check (review R2) compares every *accepted* landing against the batch we sealed at that nonce — a zombie that lands anyway is detected within one safe-finality delay and the node refuses into cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs. - L1 reorgs up to safe depth - Malicious `POST /tx` callers: malformed signatures, spoofed sender, replay across chains or apps, nonce manipulation - Malicious direct-input senders: arbitrary payload, any intent; sender authenticity is guaranteed by InputBox diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index 6f0b97c..efa4912 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -257,7 +257,7 @@ just test-watchdog-compare-harness Spawns Anvil + rollups devnet + `sequencer-devnet`, proves CM inspect SSZ at genesis matches `wallet_snapshot::encode(WalletConfig::devnet())` (same as -`tests/fixtures/wallet_snapshot_v1_empty.hex` only for Sepolia `default()`), then runs +`tests/fixtures/wallet_snapshot_empty.hex` only for Sepolia `default()`), then runs `sequencer-watchdog init` and `sequencer-watchdog tick`. When `inclusion_block` is unchanged at genesis, the runner skips L1/CM work (idle-cheap); `deposit_transfer_withdrawal_test` drives a gold batch first so compare replays real L1 inputs. @@ -303,4 +303,4 @@ cargo test -p app-core wallet_snapshot -- --test-threads=1 ``` HTTP integration for snapshot routes lives in `sequencer/tests/snapshot_endpoints.rs`. -SSZ golden bytes for the toy wallet live in `tests/fixtures/wallet_snapshot_v1_empty.{hex,bin}`. +SSZ golden bytes for the toy wallet live in `tests/fixtures/wallet_snapshot_empty.{hex,bin}`. diff --git a/examples/app-core/src/application/mod.rs b/examples/app-core/src/application/mod.rs index b1a9ee3..56a5494 100644 --- a/examples/app-core/src/application/mod.rs +++ b/examples/app-core/src/application/mod.rs @@ -9,7 +9,7 @@ mod wallet; pub use anvil_accounts::default_private_keys; pub use method::{MAX_METHOD_PAYLOAD_BYTES, Method, Transfer, Withdrawal}; pub use notice::{DepositNotice, TransferNotice}; -pub use wallet::{WalletApp, WalletConfig}; +pub use wallet::{DEVNET_SEQUENCER_ADDRESS, SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; pub use crate::wallet_snapshot::{ decode as decode_wallet_snapshot, encode as encode_wallet_snapshot, diff --git a/examples/app-core/src/application/wallet.rs b/examples/app-core/src/application/wallet.rs index 90e1af5..1afca01 100644 --- a/examples/app-core/src/application/wallet.rs +++ b/examples/app-core/src/application/wallet.rs @@ -56,6 +56,7 @@ pub struct WalletApp { balances: HashMap, nonces: HashMap, executed_input_count: u64, + last_executed_safe_block: u64, } pub const SEPOLIA_ERC20_PORTAL_ADDRESS: Address = @@ -65,27 +66,47 @@ pub const DEVNET_MOCK_USDC_ADDRESS: Address = address!("0x95d0c8A7d11342299807A2Fc19ac44C2321cCc68"); pub const SEPOLIA_SEQUENCER_ADDRESS: Address = address!("0x16d5FF3Fdd14e2a86FBA77cbcE6B3Cd9C32b8Ff3"); +/// Devnet batch-submitter / sequencer address — anvil account **9**, +/// deliberately distinct from the deployer/funder (anvil account 0). +/// +/// A dedicated submitter is a load-bearing assumption of `setup`'s detection +/// gate: step 1 refuses when the submitter's wallet nonce is +/// unsettled (`pending > safe`). The deployer has a non-zero nonce from +/// contract creations whose tail isn't safe at setup time, so reusing it as +/// the submitter false-positives. Account 9 starts at nonce 0. Kept in sync +/// with the harness submitter key (`tests/harness` uses `default_private_keys` +/// index 9) and `canonical-test`'s batch sender. pub const DEVNET_SEQUENCER_ADDRESS: Address = - address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + address!("0xa0Ee7A142d267C1f36714E4a8F75612F20a79720"); impl WalletApp { pub fn new(config: WalletConfig) -> Self { - Self::from_snapshot_parts(config, HashMap::new(), HashMap::new(), 0) + Self { + config, + balances: HashMap::new(), + nonces: HashMap::new(), + executed_input_count: 0, + last_executed_safe_block: 0, + } } + /// Reconstruct from decoded snapshot parts. Used by `crate::wallet_snapshot::decode`. pub(crate) fn from_snapshot_parts( config: WalletConfig, balances: HashMap, nonces: HashMap, executed_input_count: u64, + last_executed_safe_block: u64, ) -> Self { Self { config, balances, nonces, executed_input_count, + last_executed_safe_block, } } + // Accessors for the canonical snapshot encoder (`crate::wallet_snapshot`). pub(crate) fn config(&self) -> &WalletConfig { &self.config } @@ -108,19 +129,63 @@ impl WalletApp { &mut self.nonces } + #[cfg(test)] + pub(crate) fn set_executed_input_count(&mut self, count: u64) { + self.executed_input_count = count; + } + pub(crate) fn executed_input_count(&self) -> u64 { self.executed_input_count } - #[cfg(test)] - pub(crate) fn set_executed_input_count(&mut self, count: u64) { - self.executed_input_count = count; + pub(crate) fn last_executed_safe_block(&self) -> u64 { + self.last_executed_safe_block + } + + /// Deterministic JSON of the non-default logical state (debug only). + fn state_json(&self) -> String { + let mut balances: Vec<_> = self + .balances + .iter() + .filter(|(_, balance)| **balance != U256::ZERO) + .collect(); + balances.sort_by_key(|(address, _)| address.as_slice()); + + let mut nonces: Vec<_> = self + .nonces + .iter() + .filter(|(_, nonce)| **nonce != 0) + .collect(); + nonces.sort_by_key(|(address, _)| address.as_slice()); + + let balance_entries = balances + .into_iter() + .map(|(address, balance)| format!("\"{}\":\"{balance}\"", json_address(address))) + .collect::>() + .join(","); + let nonce_entries = nonces + .into_iter() + .map(|(address, nonce)| format!("\"{}\":{nonce}", json_address(address))) + .collect::>() + .join(","); + + format!("{{\"balances\":{{{balance_entries}}},\"nonces\":{{{nonce_entries}}}}}") } fn balance_of(&self, addr: &Address) -> U256 { *self.balances.get(addr).unwrap_or(&U256::ZERO) } + // Wallet-specific read queries (not on the Application trait — the + // sequencer never asks; app-specific query surface belongs to the app). + pub fn current_user_nonce(&self, sender: Address) -> u32 { + self.expected_nonce(&sender) + } + + pub fn current_user_balance(&self, sender: Address) -> U256 { + self.balance_of(&sender) + } + fn credit(&mut self, addr: Address, amount: U256) { let current = self.balance_of(&addr); self.balances.insert(addr, current + amount); @@ -154,35 +219,6 @@ impl WalletApp { Erc20Deposit::decode(&input.payload).map(Some) } - - fn state_json(&self) -> String { - let mut balances: Vec<_> = self - .balances - .iter() - .filter(|(_, balance)| **balance != U256::ZERO) - .collect(); - balances.sort_by_key(|(address, _)| address.as_slice()); - - let mut nonces: Vec<_> = self - .nonces - .iter() - .filter(|(_, nonce)| **nonce != 0) - .collect(); - nonces.sort_by_key(|(address, _)| address.as_slice()); - - let balance_entries = balances - .into_iter() - .map(|(address, balance)| format!("\"{}\":\"{balance}\"", json_address(address))) - .collect::>() - .join(","); - let nonce_entries = nonces - .into_iter() - .map(|(address, nonce)| format!("\"{}\":{nonce}", json_address(address))) - .collect::>() - .join(","); - - format!("{{\"balances\":{{{balance_entries}}},\"nonces\":{{{nonce_entries}}}}}") - } } fn json_address(address: &Address) -> String { @@ -198,14 +234,6 @@ impl Default for WalletApp { impl Application for WalletApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, sender: Address) -> u32 { - self.expected_nonce(&sender) - } - - fn current_user_balance(&self, sender: Address) -> U256 { - self.balance_of(&sender) - } - fn validate_user_op( &self, sender: Address, @@ -220,14 +248,14 @@ impl Application for WalletApp { }); } - // max_fee < current_fee is already checked by the trait default in + // max_fee < current_fee is already checked by the free function // validate_and_execute_user_op. No need to repeat here. - let gas_cost = sequencer_core::fee::fee_to_linear(current_fee); + let fee_cost = sequencer_core::fee::fee_to_linear(current_fee); let balance = self.balance_of(&sender); - if balance < gas_cost { - return Err(InvalidReason::InsufficientGasBalance { - required: gas_cost, + if balance < fee_cost { + return Err(InvalidReason::InsufficientFeeBalance { + required: fee_cost, available: balance, }); } @@ -235,19 +263,23 @@ impl Application for WalletApp { Ok(()) } - fn execute_valid_user_op(&mut self, user_op: &ValidUserOp) -> Result { + fn execute_valid_user_op( + &mut self, + user_op: &ValidUserOp, + safe_block: u64, + ) -> Result { let sender = user_op.sender; - let gas_cost = sequencer_core::fee::fee_to_linear(user_op.fee); + let fee_cost = sequencer_core::fee::fee_to_linear(user_op.fee); let balance = self.balance_of(&sender); - if balance < gas_cost { + if balance < fee_cost { return Err(AppError::Internal { - reason: "validated user op cannot pay gas".to_string(), + reason: "validated user op cannot pay fee".to_string(), }); } self.bump_nonce(sender); - self.balances.insert(sender, balance - gas_cost); - self.credit(self.config.sequencer_address, gas_cost); + self.balances.insert(sender, balance - fee_cost); + self.credit(self.config.sequencer_address, fee_cost); let mut outputs = Vec::new(); let method = Method::from_ssz_bytes(user_op.data.as_slice()).ok(); @@ -280,6 +312,7 @@ impl Application for WalletApp { } self.executed_input_count = self.executed_input_count.saturating_add(1); + self.last_executed_safe_block = self.last_executed_safe_block.max(safe_block); Ok(outputs) } @@ -322,6 +355,7 @@ impl Application for WalletApp { } self.executed_input_count = self.executed_input_count.saturating_add(1); + self.last_executed_safe_block = self.last_executed_safe_block.max(input.block_number); Ok(outputs) } @@ -329,16 +363,24 @@ impl Application for WalletApp { self.executed_input_count } - fn from_dump(prefix: &Path) -> Result { - let state_path = Self::state_file_in_dump(prefix); - let bytes = std::fs::read(&state_path)?; - crate::wallet_snapshot::decode(&bytes) + fn last_executed_safe_block(&self) -> u64 { + self.last_executed_safe_block } fn canonical_snapshot_bytes(&self) -> Result, AppError> { Ok(crate::wallet_snapshot::encode(self)) } + fn export_state(&self) -> Result { + Ok(self.state_json()) + } + + fn from_dump(prefix: &Path) -> Result { + let state_path = Self::state_file_in_dump(prefix); + let bytes = std::fs::read(&state_path)?; + crate::wallet_snapshot::decode(&bytes) + } + fn create_dump(&self, prefix: &Path) -> Result<(), AppError> { // `create_dir` (not `create_dir_all`) deliberately errors if the // prefix already exists. Snapshot prefixes are expected to be @@ -374,10 +416,6 @@ impl Application for WalletApp { fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } - - fn export_state(&self) -> Result { - Ok(self.state_json()) - } } #[cfg(test)] @@ -385,10 +423,8 @@ mod tests { use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; - use crate::wallet_snapshot::{SnapshotBalance, SnapshotNonce, WalletSnapshotV1}; use alloy_primitives::{Address, U256, address}; - use ssz::Encode as SszEncodeTrait; - use ssz_derive::{Decode as SszDecode, Encode as SszEncode}; + use ssz_derive::{Decode, Encode}; use types::ERC20_DEPOSIT_PREFIX_BYTES; use types::Erc20Transfer; use types::alloy_sol_types::SolCall; @@ -401,7 +437,7 @@ mod tests { #[test] fn validate_rejects_when_max_fee_below_current_fee() { - use sequencer_core::application::{Application, ExecutionOutcome}; + use sequencer_core::application::{ExecutionOutcome, validate_and_execute_user_op}; let mut app = WalletApp::new(WalletConfig::default()); let sender = Address::from_slice(&[0x11; 20]); @@ -413,10 +449,9 @@ mod tests { data: Vec::::new().into(), }; - // The max_fee < current_fee check now lives in the trait default - // (validate_and_execute_user_op), not in validate_user_op directly. - let result = app - .validate_and_execute_user_op(sender, &user_op, 2) + // The max_fee < current_fee check lives in the free function + // validate_and_execute_user_op, not in validate_user_op directly. + let result = validate_and_execute_user_op(&mut app, sender, &user_op, 2, 0) .expect("should return Ok(Invalid), not Err"); assert_eq!( result, @@ -442,7 +477,9 @@ mod tests { data: Vec::new(), }; let gas_cost = sequencer_core::fee::fee_to_linear(fee_exponent); - let outputs = app.execute_valid_user_op(&valid).expect("execute valid op"); + let outputs = app + .execute_valid_user_op(&valid, 0) + .expect("execute valid op"); assert_eq!(app.current_user_nonce(sender), 1); assert_eq!(app.current_user_balance(sender), initial_balance - gas_cost); @@ -459,24 +496,24 @@ mod tests { assert_eq!(app.current_user_balance(recipient), U256::ZERO); } - #[derive(PartialEq, Debug, SszEncode, SszDecode, Clone)] + #[derive(PartialEq, Debug, Encode, Decode, Clone)] struct LegacyDeposit { amount: U256, to: Address, } - #[derive(PartialEq, Debug, SszEncode, SszDecode, Clone)] + #[derive(PartialEq, Debug, Encode, Decode, Clone)] struct LegacyWithdrawal { amount: U256, } - #[derive(PartialEq, Debug, SszEncode, SszDecode, Clone)] + #[derive(PartialEq, Debug, Encode, Decode, Clone)] struct LegacyTransfer { amount: U256, to: Address, } - #[derive(PartialEq, Debug, SszEncode, SszDecode, Clone)] + #[derive(PartialEq, Debug, Encode, Decode, Clone)] #[ssz(enum_behaviour = "union")] enum LegacyMethod { Withdrawal(LegacyWithdrawal), @@ -502,11 +539,11 @@ mod tests { let valid = ValidUserOp { sender, fee: 0, - data: SszEncodeTrait::as_ssz_bytes(&legacy), + data: ssz::Encode::as_ssz_bytes(&legacy), }; let outputs = app - .execute_valid_user_op(&valid) + .execute_valid_user_op(&valid, 0) .expect("execute valid user op"); assert_eq!(app.current_user_nonce(sender), before_sender_nonce + 1); @@ -644,7 +681,9 @@ mod tests { })), }; - let outputs = app.execute_valid_user_op(&valid).expect("execute transfer"); + let outputs = app + .execute_valid_user_op(&valid, 0) + .expect("execute transfer"); assert_eq!( app.current_user_balance(sender), @@ -680,7 +719,7 @@ mod tests { }; let outputs = app - .execute_valid_user_op(&valid) + .execute_valid_user_op(&valid, 0) .expect("execute withdrawal"); assert_eq!( @@ -736,7 +775,7 @@ mod tests { fee: fee_exponent, data: Vec::new(), }; - app.execute_valid_user_op(&valid).expect("execute op"); + app.execute_valid_user_op(&valid, 0).expect("execute op"); assert_eq!( app.current_user_balance(sender), @@ -766,7 +805,7 @@ mod tests { fee: fee_exponent, data: Vec::new(), }; - app.execute_valid_user_op(&valid).expect("execute op"); + app.execute_valid_user_op(&valid, 0).expect("execute op"); assert_eq!( app.current_user_balance(sender), @@ -790,6 +829,7 @@ mod tests { app.nonces.insert(alice, 4); app.nonces.insert(bob, 9); app.executed_input_count = 42; + app.last_executed_safe_block = 777; let prefix = temp_dump_prefix(); app.create_dump(&prefix).expect("create dump"); @@ -813,16 +853,48 @@ mod tests { assert_eq!(restored.balances, app.balances); assert_eq!(restored.nonces, app.nonces); assert_eq!(restored.executed_input_count, app.executed_input_count); + assert_eq!( + restored.last_executed_safe_block, + app.last_executed_safe_block + ); } #[test] - fn create_dump_state_file_matches_canonical_encode() { - let app = WalletApp::new(WalletConfig::default()); - let prefix = temp_dump_prefix(); - app.create_dump(&prefix).expect("create dump"); - let on_disk = std::fs::read(WalletApp::state_file_in_dump(&prefix)).expect("read state"); - WalletApp::delete_dump(&prefix).expect("cleanup"); - assert_eq!(on_disk, crate::wallet_snapshot::encode(&app)); + fn safe_block_clock_advances_by_max_on_both_execution_paths() { + let mut app = WalletApp::new(WalletConfig::default()); + let sender = Address::from_slice(&[0x77; 20]); + app.balances.insert(sender, U256::from(10_000_u64)); + assert_eq!(app.last_executed_safe_block(), 0); + + // User op carries its covering frame's safe block. + let valid = ValidUserOp { + sender, + fee: 0, + data: Vec::new(), + }; + app.execute_valid_user_op(&valid, 100).expect("execute op"); + assert_eq!(app.last_executed_safe_block(), 100); + + // A direct input advances the clock via its own inclusion block. + let direct = sequencer_core::l2_tx::DirectInput { + sender: Address::from_slice(&[0x88; 20]), + block_number: 150, + payload: Vec::new(), + }; + app.execute_direct_input(&direct).expect("execute direct"); + assert_eq!(app.last_executed_safe_block(), 150); + + // max(): an older block must never regress the clock. A direct's + // inclusion block is <= its covering frame's safe block, so replays + // legitimately present blocks below the current clock. + let older_direct = sequencer_core::l2_tx::DirectInput { + sender: Address::from_slice(&[0x88; 20]), + block_number: 120, + payload: Vec::new(), + }; + app.execute_direct_input(&older_direct) + .expect("execute older direct"); + assert_eq!(app.last_executed_safe_block(), 150); } #[test] @@ -886,74 +958,6 @@ mod tests { } } - #[test] - fn from_snapshot_bytes_rejects_duplicate_balance_addresses() { - // Hand-crafted snapshot with two entries for the same address. - // The encoder never produces this, but the decoder must reject it - // to keep snapshot bytes canonical: without this check, multiple - // distinct byte sequences could decode to the same logical state. - let snapshot = WalletSnapshotV1 { - erc20_portal_address: [0; 20], - supported_erc20_token: [0; 20], - sequencer_address: [0; 20], - balances: vec![ - SnapshotBalance { - address: [1; 20], - balance_be: [0; 32], - }, - SnapshotBalance { - address: [1; 20], - balance_be: [0; 32], - }, - ], - nonces: vec![], - executed_input_count: 0, - }; - let bytes = ssz::Encode::as_ssz_bytes(&snapshot); - - let err = - crate::wallet_snapshot::decode(&bytes).expect_err("duplicate balance should reject"); - match err { - AppError::Internal { reason } => assert!( - reason.contains("duplicate balance"), - "expected duplicate-balance error, got: {reason}" - ), - other => panic!("expected Internal duplicate error, got {other:?}"), - } - } - - #[test] - fn from_snapshot_bytes_rejects_duplicate_nonce_addresses() { - let snapshot = WalletSnapshotV1 { - erc20_portal_address: [0; 20], - supported_erc20_token: [0; 20], - sequencer_address: [0; 20], - balances: vec![], - nonces: vec![ - SnapshotNonce { - address: [1; 20], - nonce: 0, - }, - SnapshotNonce { - address: [1; 20], - nonce: 1, - }, - ], - executed_input_count: 0, - }; - let bytes = ssz::Encode::as_ssz_bytes(&snapshot); - - let err = - crate::wallet_snapshot::decode(&bytes).expect_err("duplicate nonce should reject"); - match err { - AppError::Internal { reason } => assert!( - reason.contains("duplicate nonce"), - "expected duplicate-nonce error, got: {reason}" - ), - other => panic!("expected Internal duplicate error, got {other:?}"), - } - } - #[test] fn state_file_in_dump_lives_inside_prefix() { let prefix = PathBuf::from("/tmp/example-prefix"); @@ -977,30 +981,4 @@ mod tests { path.push(format!("wallet-dump-{}-{nanos}-{n}", std::process::id())); path } - - #[test] - fn export_state_is_deterministic_and_omits_defaults() { - let mut app = WalletApp::new(WalletConfig::default()); - let high = address!("0xffffffffffffffffffffffffffffffffffffffff"); - let low = address!("0x1111111111111111111111111111111111111111"); - app.balances.insert(high, U256::from(20_u64)); - app.balances.insert(low, U256::from(10_u64)); - app.balances.insert(Address::ZERO, U256::ZERO); - app.nonces.insert(high, 2); - app.nonces.insert(low, 1); - app.nonces.insert(Address::ZERO, 0); - - assert_eq!( - app.export_state().expect("export state"), - concat!( - "{\"balances\":{", - "\"0x1111111111111111111111111111111111111111\":\"10\",", - "\"0xffffffffffffffffffffffffffffffffffffffff\":\"20\"", - "},\"nonces\":{", - "\"0x1111111111111111111111111111111111111111\":1,", - "\"0xffffffffffffffffffffffffffffffffffffffff\":2", - "}}" - ) - ); - } } diff --git a/examples/app-core/src/wallet_snapshot.rs b/examples/app-core/src/wallet_snapshot.rs index 4ec96f3..1c6fa2e 100644 --- a/examples/app-core/src/wallet_snapshot.rs +++ b/examples/app-core/src/wallet_snapshot.rs @@ -7,7 +7,7 @@ //! [`WalletApp::create_dump`](crate::application::wallet::WalletApp::create_dump), //! CM `inspect_state`, and the watchdog's `/finalized_state` byte compare. //! -//! Golden bytes: `tests/fixtures/wallet_snapshot_v1_empty.{hex,bin}` (shared with +//! Golden bytes: `tests/fixtures/wallet_snapshot_empty.{hex,bin}` (shared with //! Rust and Lua parity tests). use std::collections::HashMap; @@ -32,13 +32,14 @@ pub struct SnapshotNonce { } #[derive(Debug, Clone, PartialEq, Eq, SszEncode, SszDecode)] -pub struct WalletSnapshotV1 { +pub struct WalletSnapshot { pub erc20_portal_address: [u8; 20], pub supported_erc20_token: [u8; 20], pub sequencer_address: [u8; 20], pub balances: Vec, pub nonces: Vec, pub executed_input_count: u64, + pub last_executed_safe_block: u64, } /// Deterministic SSZ bytes for `app`'s logical state (sorted map entries). @@ -61,20 +62,21 @@ pub fn encode(app: &WalletApp) -> Vec { .collect(); nonces.sort_unstable_by_key(|entry| entry.address); - WalletSnapshotV1 { + WalletSnapshot { erc20_portal_address: app.config().erc20_portal_address.into_array(), supported_erc20_token: app.config().supported_erc20_token.into_array(), sequencer_address: app.config().sequencer_address.into_array(), balances, nonces, executed_input_count: app.executed_input_count(), + last_executed_safe_block: app.last_executed_safe_block(), } .as_ssz_bytes() } /// Rehydrate a [`WalletApp`] from SSZ snapshot bytes. pub fn decode(bytes: &[u8]) -> Result { - let decoded = WalletSnapshotV1::from_ssz_bytes(bytes).map_err(|e| AppError::Internal { + let decoded = WalletSnapshot::from_ssz_bytes(bytes).map_err(|e| AppError::Internal { reason: format!("snapshot decode failed: {e:?}"), })?; @@ -108,6 +110,7 @@ pub fn decode(bytes: &[u8]) -> Result { balances, nonces, decoded.executed_input_count, + decoded.last_executed_safe_block, )) } @@ -136,10 +139,7 @@ mod tests { #[test] fn encode_default_wallet_matches_golden_vector() { let app = WalletApp::new(WalletConfig::default()); - assert_eq!( - encode(&app), - read_fixture_hex("wallet_snapshot_v1_empty.hex") - ); + assert_eq!(encode(&app), read_fixture_hex("wallet_snapshot_empty.hex")); } #[test] diff --git a/examples/canonical-app/Cargo.toml b/examples/canonical-app/Cargo.toml index 44d35e5..cf1f5b3 100644 --- a/examples/canonical-app/Cargo.toml +++ b/examples/canonical-app/Cargo.toml @@ -14,8 +14,6 @@ app-core = { path = "../app-core" } sequencer-core = { path = "../../sequencer-core" } trolley = { workspace = true } alloy-primitives = { version = "1.4.1", features = ["serde", "k256"] } -alloy-sol-types = "1.4.1" -ssz = { package = "ethereum_ssz", version = "0.10" } types = { workspace = true } [dev-dependencies] diff --git a/examples/canonical-app/src/bin/canonical-app-devnet.rs b/examples/canonical-app/src/bin/canonical-app-devnet.rs index 841aa3f..3480a21 100644 --- a/examples/canonical-app/src/bin/canonical-app-devnet.rs +++ b/examples/canonical-app/src/bin/canonical-app-devnet.rs @@ -1,12 +1,12 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use app_core::application::{WalletApp, WalletConfig}; +use app_core::application::{DEVNET_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; use canonical_app::{SchedulerConfig, run_scheduler_forever}; use trolley::cmt::RollupCmt; fn main() { let rollup = RollupCmt::try_new().expect("failed to initialize rollup"); let app = WalletApp::new(WalletConfig::devnet()); - run_scheduler_forever(rollup, app, SchedulerConfig::devnet()); + run_scheduler_forever(rollup, app, SchedulerConfig::new(DEVNET_SEQUENCER_ADDRESS)); } diff --git a/examples/canonical-app/src/bin/canonical-app-sepolia.rs b/examples/canonical-app/src/bin/canonical-app-sepolia.rs index b55c517..f0139bd 100644 --- a/examples/canonical-app/src/bin/canonical-app-sepolia.rs +++ b/examples/canonical-app/src/bin/canonical-app-sepolia.rs @@ -1,12 +1,12 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use app_core::application::{WalletApp, WalletConfig}; +use app_core::application::{SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; use canonical_app::{SchedulerConfig, run_scheduler_forever}; use trolley::cmt::RollupCmt; fn main() { let rollup = RollupCmt::try_new().expect("failed to initialize rollup"); let app = WalletApp::new(WalletConfig::sepolia()); - run_scheduler_forever(rollup, app, SchedulerConfig::sepolia()); + run_scheduler_forever(rollup, app, SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS)); } diff --git a/examples/canonical-app/src/lib.rs b/examples/canonical-app/src/lib.rs index 13141b1..845c33e 100644 --- a/examples/canonical-app/src/lib.rs +++ b/examples/canonical-app/src/lib.rs @@ -3,6 +3,4 @@ pub mod scheduler; -pub use scheduler::{ - DEVNET_SEQUENCER_ADDRESS, SEPOLIA_SEQUENCER_ADDRESS, SchedulerConfig, run_scheduler_forever, -}; +pub use scheduler::{SchedulerConfig, run_scheduler_forever}; diff --git a/examples/canonical-app/src/scheduler/mod.rs b/examples/canonical-app/src/scheduler/mod.rs index cd12f41..793566d 100644 --- a/examples/canonical-app/src/scheduler/mod.rs +++ b/examples/canonical-app/src/scheduler/mod.rs @@ -1,34 +1,40 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -mod core; - -pub use core::{ - DEVNET_SEQUENCER_ADDRESS, SEPOLIA_SEQUENCER_ADDRESS, STATE_INSPECT_QUERY, SchedulerConfig, -}; +//! Canonical-app harness around the scheduler library. +//! +//! The pure scheduler fold lives in [`sequencer_core::scheduler`]; this module +//! is the I/O shell that drives it inside the RISC-V app. It pulls inputs off +//! the `trolley` rollup, coerces host `U256` metadata into the scheduler's +//! `u64` domain, feeds them through [`Scheduler::process_input`], and emits the +//! app's notices/vouchers/reports back to the rollup. +use alloy_primitives::U256; use sequencer_core::application::AppOutput; use sequencer_core::application::Application; +use sequencer_core::scheduler::{ + InspectError, ProcessOutcome, Scheduler, SchedulerInput, input_domain, +}; use trolley::{Rollup, RollupRequest}; use types::{Notice, Voucher}; +pub use sequencer_core::scheduler::{STATE_INSPECT_QUERY, SchedulerConfig}; + pub fn run_scheduler_forever( mut rollup: R, app: A, scheduler_config: SchedulerConfig, ) -> ! { - let mut scheduler = core::Scheduler::new(app, scheduler_config); + let mut scheduler = Scheduler::new(app, scheduler_config); loop { match rollup.next_input() { Ok(RollupRequest::Advance { metadata, payload }) => { - let inclusion_block = core::block_to_u64(metadata.block_number); - let domain = core::input_domain( - core::chain_id_to_u64(metadata.chain_id), - metadata.app_contract, - ); + let inclusion_block = block_to_u64(metadata.block_number); + let domain = + input_domain(chain_id_to_u64(metadata.chain_id), metadata.app_contract); - let input = core::SchedulerInput { + let input = SchedulerInput { sender: metadata.msg_sender, inclusion_block, domain, @@ -40,7 +46,7 @@ pub fn run_scheduler_forever( emit_app_output(&mut rollup, output) .unwrap_or_else(|err| panic!("scheduler failed to emit app output: {err}")); } - if matches!(result.outcome, core::ProcessOutcome::BatchRejected(_)) { + if matches!(result.outcome, ProcessOutcome::BatchRejected(_)) { rollup .emit_report(b"scheduler dropped invalid batch") .unwrap_or_else(|err| { @@ -54,10 +60,8 @@ pub fn run_scheduler_forever( // structured error report and keep serving, as it did before. let report = match scheduler.inspect_state(&payload) { Ok(bytes) => bytes, - Err(core::InspectError::UnsupportedQuery) => { - b"unsupported inspect query".to_vec() - } - Err(core::InspectError::Application(reason)) => { + Err(InspectError::UnsupportedQuery) => b"unsupported inspect query".to_vec(), + Err(InspectError::Application(reason)) => { format!("inspect failed: {reason}").into_bytes() } }; @@ -93,13 +97,26 @@ fn emit_app_output(rollup: &mut R, output: &AppOutput) -> trolley::Ro } } +/// Coerce a host `U256` block number to the scheduler's `u64` domain. Solidity +/// exposes block numbers as `uint256`; a value that does not fit `u64` is a +/// malformed host input for this prototype. Host-side, so it stays in the +/// harness rather than the pure scheduler library. +fn block_to_u64(block: U256) -> u64 { + u64::try_from(block).expect("block number does not fit u64") +} + +/// Coerce a host `U256` chain id to `u64` (same rationale as [`block_to_u64`]). +fn chain_id_to_u64(chain_id: U256) -> u64 { + u64::try_from(chain_id).expect("chain id does not fit u64") +} + #[cfg(test)] mod tests { use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use alloy_primitives::{Address, U256}; - use app_core::application::{WalletApp, WalletConfig}; + use app_core::application::{SEPOLIA_SEQUENCER_ADDRESS, WalletApp, WalletConfig}; use trolley::{InputMetadata, RollupError}; use super::*; @@ -187,7 +204,7 @@ mod tests { run_scheduler_forever( rollup, WalletApp::new(WalletConfig::default()), - SchedulerConfig::default(), + SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), ) })); @@ -222,7 +239,7 @@ mod tests { run_scheduler_forever( rollup, WalletApp::new(WalletConfig::default()), - SchedulerConfig::default(), + SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), ) })); @@ -241,7 +258,7 @@ mod tests { #[test] fn run_scheduler_emits_report_for_invalid_batch_before_rollup_error() { - let sequencer = SchedulerConfig::default().sequencer_address; + let sequencer = SEPOLIA_SEQUENCER_ADDRESS; let invalid_batch_input = RollupRequest::Advance { metadata: metadata(sequencer, 10), payload: vec![0xFF, 0xEE, 0xDD], @@ -257,7 +274,7 @@ mod tests { run_scheduler_forever( rollup, WalletApp::new(WalletConfig::default()), - SchedulerConfig::default(), + SchedulerConfig::new(SEPOLIA_SEQUENCER_ADDRESS), ) })); diff --git a/examples/canonical-test/src/main.rs b/examples/canonical-test/src/main.rs index 32d57f6..214cda4 100644 --- a/examples/canonical-test/src/main.rs +++ b/examples/canonical-test/src/main.rs @@ -4,9 +4,10 @@ use std::path::PathBuf; use app_core::application::{ - DepositNotice, Method, Transfer, TransferNotice, WalletConfig, Withdrawal, + DEVNET_SEQUENCER_ADDRESS, DepositNotice, Method, Transfer, TransferNotice, WalletConfig, + Withdrawal, }; -use canonical_app::{DEVNET_SEQUENCER_ADDRESS, SchedulerConfig}; +use canonical_app::SchedulerConfig; use k256::ecdsa::SigningKey; use k256::ecdsa::signature::hazmat::PrehashSigner; use sequencer_core::batch::{Batch, Frame, WireUserOp}; @@ -51,7 +52,8 @@ pub fn scheduler_rejected_batch_does_not_consume_nonce() -> TestResult { #[testsi::test_dapp(kind("scheduler"))] pub fn scheduler_stale_batch_is_skipped_without_consuming_nonce() -> TestResult { let mut machine = devnet_machine()?; - let stale_trigger_block = SchedulerConfig::devnet().max_wait_blocks as usize + 1; + let stale_trigger_block = + SchedulerConfig::new(DEVNET_SEQUENCER_ADDRESS).max_wait_blocks as usize + 1; // Stale batch (nonce 0, safe_block 1, inclusion block > max_wait_blocks) → skipped silently. let (outputs, reports) = machine.advance_state(batch_input( @@ -219,6 +221,86 @@ pub fn scheduler_emits_withdrawal_voucher_from_guest() -> TestResult { Ok(()) } +/// T3: the guest's fee arithmetic must agree with the host's `fee_to_linear`. +/// Every other scheduler test runs frames at `fee_price: 0`, so the guest's gas +/// charging and max-fee skip never ran with a nonzero fee — a guest-side +/// divergence would be silent. This drives one nonzero-fee frame and pins the +/// charged gas to the host's value from BOTH sides, plus the below-fee skip: +/// - Alice transfers exactly `deposit - gas`: affordable iff guest gas <= host. +/// - Carol transfers `deposit - gas + 1`: affordable iff guest gas < host. +/// - Dave's op has `max_fee < fee_price`: must be skipped (below the frame fee). +/// A correct guest emits exactly one transfer notice (Alice's) — so guest gas == +/// host `fee_to_linear(fee_price)` bit-for-bit, and the max-fee skip holds. +#[testsi::test_dapp(kind("scheduler"))] +pub fn scheduler_fee_arithmetic_matches_host_from_guest() -> TestResult { + let mut machine = devnet_machine()?; + let token = WalletConfig::devnet().supported_erc20_token; + let bob = address!("0x8888888888888888888888888888888888888888"); + let fee_price: u16 = 200; + let gas = sequencer_core::fee::fee_to_linear(fee_price); + let deposit = U256::from(10_000_u64); + + let alice_key = signing_key(11); + let alice = address_from_signing_key(&alice_key); + let carol_key = signing_key(12); + let carol = address_from_signing_key(&carol_key); + let dave_key = signing_key(13); + let dave = address_from_signing_key(&dave_key); + + // Fund all three; one drain batch executes the three pending deposits. + for who in [alice, carol, dave] { + let (outputs, reports) = machine.advance_state(portal_input(10, token, who, deposit))?; + assert_no_outputs_or_reports(&outputs, &reports); + } + let (outputs, reports) = + machine.advance_state(batch_input(10, batch_with_safe_blocks(0, &[10])))?; + assert!(reports.is_empty(), "drain reports: {reports:?}"); + assert_eq!( + outputs.list().len(), + 3, + "expected three deposit notices, got {:?}", + outputs.list() + ); + + let exact = deposit - gas; + let transfer = + |amount| ssz::Encode::as_ssz_bytes(&Method::Transfer(Transfer { amount, to: bob })); + let alice_exact = signed_user_op_with_fee(&alice_key, 0, fee_price, transfer(exact)); + let carol_over = signed_user_op_with_fee( + &carol_key, + 0, + fee_price, + transfer(exact + U256::from(1_u64)), + ); + let dave_below_fee = + signed_user_op_with_fee(&dave_key, 0, fee_price - 1, transfer(U256::from(1_u64))); + + let (outputs, reports) = machine.advance_state(batch_input( + 11, + batch_with_frame_fee( + 1, + 10, + fee_price, + vec![alice_exact, carol_over, dave_below_fee], + ), + ))?; + assert!(reports.is_empty(), "fee-frame reports: {reports:?}"); + assert_eq!( + outputs.list().len(), + 1, + "expected exactly one transfer notice (Alice's exact transfer): Carol's \ + over-by-one must be unaffordable (guest charged the full host gas) and \ + Dave's below-fee op must be skipped, got {:?}", + outputs.list() + ); + let notice = outputs[0].expect_notice(); + let decoded = TransferNotice::abi_decode(¬ice.payload).expect("decode transfer notice"); + assert_eq!(decoded.sender, alice); + assert_eq!(decoded.recipient, bob); + assert_eq!(decoded.amount, exact); + Ok(()) +} + fn devnet_machine() -> Result> { let machine_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../canonical-app/out/canonical-machine-image"); @@ -246,9 +328,18 @@ fn address_from_signing_key(signing_key: &SigningKey) -> Address { } fn signed_user_op(signing_key: &SigningKey, nonce: u32, data: Vec) -> WireUserOp { + signed_user_op_with_fee(signing_key, nonce, 0, data) +} + +fn signed_user_op_with_fee( + signing_key: &SigningKey, + nonce: u32, + max_fee: u16, + data: Vec, +) -> WireUserOp { let user_op = UserOp { nonce, - max_fee: 0, + max_fee, data: data.clone().into(), }; let signing_hash = user_op.eip712_signing_hash(&input_domain()); @@ -270,7 +361,7 @@ fn signed_user_op(signing_key: &SigningKey, nonce: u32, data: Vec) -> WireUs WireUserOp { nonce, - max_fee: 0, + max_fee, data, signature: signature.as_bytes().to_vec(), } @@ -317,12 +408,21 @@ fn batch_with_safe_blocks(nonce: u64, safe_blocks: &[u64]) -> Batch { } fn batch_with_frame(nonce: u64, safe_block: u64, user_ops: Vec) -> Batch { + batch_with_frame_fee(nonce, safe_block, 0, user_ops) +} + +fn batch_with_frame_fee( + nonce: u64, + safe_block: u64, + fee_price: u16, + user_ops: Vec, +) -> Batch { Batch { nonce, frames: vec![Frame { user_ops, safe_block, - fee_price: 0, + fee_price, }], } } diff --git a/examples/wallet-sequencer/Cargo.toml b/examples/wallet-sequencer/Cargo.toml new file mode 100644 index 0000000..9a1c0fb --- /dev/null +++ b/examples/wallet-sequencer/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "wallet-sequencer" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Sequencer binary for the placeholder wallet app" +homepage.workspace = true +repository.workspace = true +readme.workspace = true +authors.workspace = true +default-run = "wallet-sequencer" + +[dependencies] +app-core = { path = "../app-core" } +sequencer = { path = "../../sequencer" } +tokio = { version = "1.35", features = ["macros", "rt-multi-thread"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/sequencer/src/main.rs b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs similarity index 62% rename from sequencer/src/main.rs rename to examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs index 6bf08f9..2914cc3 100644 --- a/sequencer/src/main.rs +++ b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs @@ -2,18 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) use app_core::application::{WalletApp, WalletConfig}; -use clap::Parser; -use sequencer::{RunConfig, run}; use tracing_subscriber::EnvFilter; #[tokio::main] -async fn main() -> Result<(), sequencer::RunError> { +async fn main() -> std::process::ExitCode { tracing_subscriber::fmt() .with_env_filter( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), ) .init(); - let config = RunConfig::parse(); - run(WalletApp::new(WalletConfig::default()), config).await + // `setup` is the only subcommand that constructs a genesis app; the + // closure runs only on that path. + sequencer::run_main(|| WalletApp::new(WalletConfig::devnet())).await } diff --git a/examples/wallet-sequencer/src/main.rs b/examples/wallet-sequencer/src/main.rs new file mode 100644 index 0000000..c33d1d3 --- /dev/null +++ b/examples/wallet-sequencer/src/main.rs @@ -0,0 +1,18 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use app_core::application::{WalletApp, WalletConfig}; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> std::process::ExitCode { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + // `setup` is the only subcommand that constructs a genesis app; the + // closure runs only on that path. + sequencer::run_main(|| WalletApp::new(WalletConfig::default())).await +} diff --git a/justfile b/justfile index 185df59..47acbfe 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,12 @@ set shell := ["bash", "-euo", "pipefail", "-c"] +# Nested justfiles as modules: `just watchdog `, `just canonical `, +# `just bench ` (also `just --list`). The curated recipes below are +# top-level shortcuts for the common cross-cutting operations. +mod watchdog 'watchdog/justfile' +mod canonical 'examples/canonical-app/justfile' +mod bench 'tests/benchmarks/justfile' + default: @just --list @@ -13,27 +20,27 @@ test: cargo test --workspace test-watchdog: - just -f watchdog/justfile test + just watchdog test test-watchdog-e2e: - just -f watchdog/justfile test-e2e + just watchdog test-e2e # Verify divergence signal via main.lua (drill exits 2 like production). test-watchdog-divergence-drill: watchdog-lua-deps - @just -f watchdog/justfile test-divergence-drill + @just watchdog test-divergence-drill # Build lcurl (lua-cURLv3) into .deps/lua; JSON is pure Lua under watchdog/third_party/. watchdog-lua-deps: - @just -f watchdog/justfile lua-deps + @just watchdog lua-deps -# Anvil + rollups + sequencer-devnet; prints CARTESI_WATCHDOG_* exports until Ctrl+C. +# Anvil + rollups + wallet-sequencer-devnet; prints CARTESI_WATCHDOG_* exports until Ctrl+C. devnet-for-watchdog: setup ensure-machine-image - cargo build -p sequencer --bin sequencer-devnet + cargo build -p wallet-sequencer --bin wallet-sequencer-devnet cargo build -p rollups-e2e --bin devnet-stack cargo run -p rollups-e2e --bin devnet-stack test-watchdog-compare-harness: setup watchdog-lua-deps ensure-machine-image - cargo build -p sequencer --bin sequencer-devnet -p rollups-e2e --bin rollups-e2e + cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p rollups-e2e --bin rollups-e2e cargo run -p rollups-e2e --bin rollups-e2e -- watchdog_genesis_compare_test --exact --nocapture # Run sequencer tests sequentially so partition static config (init) is not shared across parallel tests. @@ -45,43 +52,40 @@ test-sequencer: test-rollups-e2e: setup ensure-machine-image ensure-sepolia-machine-image just watchdog-lua-deps - cargo build -p sequencer --bin sequencer-devnet -p rollups-e2e --bin rollups-e2e + cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p rollups-e2e --bin rollups-e2e cargo run -p rollups-e2e --bin rollups-e2e ensure-machine-image: - @test -d examples/canonical-app/out/canonical-machine-image || just canonical-build-machine-image + @test -d examples/canonical-app/out/canonical-machine-image || just canonical build-machine-image ensure-sepolia-machine-image: - @test -d examples/canonical-app/out/canonical-machine-image-sepolia || just canonical-build-machine-image-sepolia - -bench target="all": - just -f tests/benchmarks/justfile {{target}} + @test -d examples/canonical-app/out/canonical-machine-image-sepolia || just canonical build-machine-image-sepolia setup: - just -f examples/canonical-app/justfile download-deps - just -f tests/benchmarks/justfile setup + just canonical download-deps + just bench setup just watchdog-lua-deps doctor: - just -f watchdog/justfile doctor + just watchdog doctor canonical-build-machine-image: - just -f examples/canonical-app/justfile build-machine-image + just canonical build-machine-image canonical-build-machine-image-sepolia: - just -f examples/canonical-app/justfile build-machine-image-sepolia + just canonical build-machine-image-sepolia canonical-test-guest: - just -f examples/canonical-app/justfile test-guest + just canonical test-guest canonical-print-build-hashes: - just -f examples/canonical-app/justfile print-build-hashes + just canonical print-build-hashes clean: cargo clean rm -rf sequencer-data - just -f examples/canonical-app/justfile clean - just -f tests/benchmarks/justfile clean + just canonical clean + just bench clean fmt: cargo fmt --all @@ -102,4 +106,4 @@ ci: run addr="127.0.0.1:3000" data_dir="sequencer-data": rm -rf {{data_dir}} - CARTESI_SEQUENCER_HTTP_ADDR={{addr}} CARTESI_SEQUENCER_DATA_DIR={{data_dir}} cargo run -p sequencer --release + CARTESI_SEQUENCER_HTTP_ADDR={{addr}} CARTESI_SEQUENCER_DATA_DIR={{data_dir}} cargo run -p wallet-sequencer --release diff --git a/scripts/generate-release-manifest.sh b/scripts/generate-release-manifest.sh index 2caea61..b7474b0 100755 --- a/scripts/generate-release-manifest.sh +++ b/scripts/generate-release-manifest.sh @@ -54,8 +54,8 @@ set +a artifacts_json="$(cat < { write!(f, "max fee {max_fee} below base fee {base_fee}") } - Self::InsufficientGasBalance { + Self::InsufficientFeeBalance { required, available, } => { write!( f, - "insufficient balance for gas: required {required}, available {available}" + "insufficient balance for fee: required {required}, available {available}" ) } } @@ -87,10 +90,11 @@ impl fmt::Display for InvalidReason { pub trait Application: Send + Sized { const MAX_METHOD_PAYLOAD_BYTES: usize; - fn current_user_nonce(&self, sender: Address) -> u32; - - fn current_user_balance(&self, sender: Address) -> U256; - + /// Pure validation predicate over current app state: nonce match + /// (user replay protection) and fee-balance coverage. Must not + /// mutate state. The protocol-level `max_fee >= current_fee` guard + /// is NOT this method's job — [`validate_and_execute_user_op`] + /// enforces it before calling here. fn validate_user_op( &self, sender: Address, @@ -98,43 +102,37 @@ pub trait Application: Send + Sized { current_fee: u16, ) -> Result<(), InvalidReason>; - fn execute_valid_user_op(&mut self, user_op: &ValidUserOp) -> Result; - - fn validate_and_execute_user_op( + /// Execute a validated user op. `safe_block` is the covering frame's + /// safe block; the impl must advance its safe-block clock with it: + /// `clock = max(clock, safe_block)` (see + /// [`Application::last_executed_safe_block`]). + fn execute_valid_user_op( &mut self, - sender: Address, - user_op: &UserOp, - current_fee: u16, - ) -> Result { - // Protocol invariant: max_fee must cover the current frame fee. - // Enforced here so every Application impl inherits it. - if user_op.max_fee < current_fee { - return Ok(ExecutionOutcome::Invalid(InvalidReason::InvalidMaxFee { - max_fee: user_op.max_fee, - base_fee: current_fee, - })); - } - - if let Err(reason) = self.validate_user_op(sender, user_op, current_fee) { - return Ok(ExecutionOutcome::Invalid(reason)); - } - - let valid = ValidUserOp { - sender, - fee: current_fee, - data: user_op.data.to_vec(), - }; - let outputs = self.execute_valid_user_op(&valid)?; - Ok(ExecutionOutcome::Included { outputs }) - } - - fn execute_direct_input(&mut self, _input: &DirectInput) -> Result { - Ok(Vec::new()) - } - - fn executed_input_count(&self) -> u64 { - 0 - } + user_op: &ValidUserOp, + safe_block: u64, + ) -> Result; + + /// Required (no default): deposits are direct-input-only, so a silent + /// no-op impl would strand every deposit on L1 with no L2 credit. + /// The impl must advance its safe-block clock with + /// `input.block_number` (the direct's L1 inclusion block): + /// `clock = max(clock, block_number)`. + fn execute_direct_input(&mut self, input: &DirectInput) -> Result; + + /// The app's safe-block clock: the maximum block carried by any input + /// this instance has executed (frame safe blocks for user ops, L1 + /// inclusion blocks for direct inputs), or 0 if nothing executed. + /// Carried in execution — not a setter — so an app cannot execute and + /// forget to advance it. Recovery reads this as `A`, the safe block a + /// checkpoint state reflects; it must therefore survive + /// `create_dump`/`from_dump` round-trips. + fn last_executed_safe_block(&self) -> u64; + + /// Count of executed inputs (user ops + direct inputs). Diagnostic + /// seam: replay/catch-up tests compare live vs replayed apps with it. + /// Required (no default) for the same reason as + /// [`Application::execute_direct_input`]. + fn executed_input_count(&self) -> u64; // -------- snapshot / dump lifecycle -------- // @@ -202,3 +200,40 @@ pub trait Application: Send + Sized { }) } } + +/// The single entry point for executing a user op against an app: protocol +/// guard, then app validation, then execution. +/// +/// Deliberately a free function, not a trait method: an overridable default +/// would let an `Application` impl skip the protocol-level +/// `max_fee >= current_fee` invariant. As a free function the guard is +/// non-bypassable by construction. Both consumers — the inclusion lane and +/// the canonical scheduler — must execute user ops through here; agreement +/// between them is the system's most load-bearing invariant. +pub fn validate_and_execute_user_op( + app: &mut A, + sender: Address, + user_op: &UserOp, + current_fee: u16, + safe_block: u64, +) -> Result { + // Protocol invariant: max_fee must cover the current frame fee. + if user_op.max_fee < current_fee { + return Ok(ExecutionOutcome::Invalid(InvalidReason::InvalidMaxFee { + max_fee: user_op.max_fee, + base_fee: current_fee, + })); + } + + if let Err(reason) = app.validate_user_op(sender, user_op, current_fee) { + return Ok(ExecutionOutcome::Invalid(reason)); + } + + let valid = ValidUserOp { + sender, + fee: current_fee, + data: user_op.data.to_vec(), + }; + let outputs = app.execute_valid_user_op(&valid, safe_block)?; + Ok(ExecutionOutcome::Included { outputs }) +} diff --git a/sequencer-core/src/batch.rs b/sequencer-core/src/batch.rs index 2f3fa4b..9c9104b 100644 --- a/sequencer-core/src/batch.rs +++ b/sequencer-core/src/batch.rs @@ -33,8 +33,10 @@ use ssz_derive::{Decode, Encode}; // linear gas price before converting to log space. // --------------------------------------------------------------------------- -/// Batch submissions are sent as raw `ssz(Batch)` with no tag; classification at L1 is by -/// attempting SSZ decode, and at the rollup by msg_sender. +/// Batch submissions are sent as raw `ssz(Batch)` with no tag; classification is +/// by `msg_sender` everywhere (sender == batch-submitter address ⇒ batch, any +/// other sender ⇒ direct input) — never by decode-attempt. See AGENTS.md +/// "InputBox payload classification". #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] pub struct Batch { diff --git a/sequencer-core/src/lib.rs b/sequencer-core/src/lib.rs index 3f645ee..fe615b5 100644 --- a/sequencer-core/src/lib.rs +++ b/sequencer-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod broadcast; pub mod fee; pub mod l2_tx; pub mod protocol; +pub mod scheduler; pub mod user_op; /// Maximum number of L1 blocks a batch can wait before the scheduler considers it stale. diff --git a/sequencer-core/src/protocol.rs b/sequencer-core/src/protocol.rs index f4ac2d7..e12e726 100644 --- a/sequencer-core/src/protocol.rs +++ b/sequencer-core/src/protocol.rs @@ -218,6 +218,17 @@ impl ProtocolTiming { /// Rejection paths (wrong sender, SSZ decode failure, stale by inclusion, /// nonce mismatch) return `None` without advancing — matching what the /// scheduler does on-chain. + /// + /// **Deliberately omitted structural rejects.** The canonical fold's + /// `batch_reject_reason_for_block` additionally rejects a batch whose frames + /// have a `safe_block` above the inclusion block + /// (`SafeBlockAboveInclusionBlock`) or that decrease across frames + /// (`NonMonotonicSafeBlocks`). This predicate does not re-check those: a + /// well-formed batch submitter never builds such a batch, so they are + /// reachable only under a buggy or forked submitter — already outside the + /// duality boundary. The I1 duality test pins this as a *documented* + /// asymmetry; mirror those two checks here if the submitter ever stops being + /// trusted to be well-formed. pub fn scheduler_accepts( &self, batch_submitter: Address, @@ -254,6 +265,42 @@ pub fn age_exceeds(reference_block: u64, first_frame_safe_block: u64, threshold: reference_block.saturating_sub(first_frame_safe_block) >= threshold } +/// Advance `expected` by greedily consuming any matching observed nonce. +/// +/// `observed_nonces` is the stream of **batch nonces** (from the SSZ payload) +/// decoded from `InputAdded` events sent by our batch-submitter EOA, in L1 +/// event order. Because L1 mines txs from a single EOA in strict wallet-nonce +/// order, this stream is naturally gap-less at the wallet-nonce level: +/// tx[k]'s event cannot appear on-chain without tx[k-1]'s event, and the +/// observed batch nonce sequence therefore mirrors our submission order. +/// +/// Batch nonces themselves (unlike wallet nonces) CAN repeat across recovery +/// generations — e.g., after a cascade, a fresh batch reuses its invalidated +/// predecessor's nonce. That's why we still match on equality rather than +/// trusting a sort: in a post-recovery window, the same batch nonce can be +/// observed twice (once from the invalidated generation, once from the new +/// one), and we only want to advance once. +/// +/// Under the wallet-nonce ordering above, once the next `expected` doesn't +/// appear in the stream the frontier naturally stops advancing — the gap +/// means the scheduler hasn't seen that nonce on-chain yet (or observed it at +/// a different wallet nonce from an earlier generation). +/// +/// Co-located here next to [`ProtocolTiming::scheduler_accepts`] so all +/// scheduler-mirroring acceptance logic lives at the library seam (the +/// off-chain mirror of the canonical scheduler's batch-nonce fold). +pub fn advance_expected_batch_nonce( + mut expected: u64, + observed_nonces: impl IntoIterator, +) -> u64 { + for nonce in observed_nonces { + if nonce == expected { + expected = expected.saturating_add(1); + } + } + expected +} + /// Borrowed view of one safe-input row, in the shape scheduler_accepts needs. /// Using a borrowed payload avoids copying during iteration. #[derive(Debug, Clone, Copy)] @@ -597,4 +644,20 @@ mod tests { }; assert!(timing().scheduler_accepts(SUBMITTER, input, 0).is_some()); } + + #[test] + fn advance_expected_batch_nonce_matches_scheduler_nonce_rule() { + use super::advance_expected_batch_nonce; + assert_eq!(advance_expected_batch_nonce(0, Vec::::new()), 0); + assert_eq!(advance_expected_batch_nonce(0, vec![0, 1, 2]), 3); + assert_eq!(advance_expected_batch_nonce(0, vec![0, 2, 3]), 1); + assert_eq!(advance_expected_batch_nonce(0, vec![1, 2, 3]), 0); + assert_eq!(advance_expected_batch_nonce(0, vec![0, 1, 1, 2]), 3); + assert_eq!( + advance_expected_batch_nonce(0, vec![6, 4, 3, 2, 2, 0, 1]), + 2 + ); + assert_eq!(advance_expected_batch_nonce(0, vec![0, 2, 1]), 2); + assert_eq!(advance_expected_batch_nonce(2, vec![2, 3]), 4); + } } diff --git a/sequencer-core/src/scheduler/fold.rs b/sequencer-core/src/scheduler/fold.rs new file mode 100644 index 0000000..0b599a0 --- /dev/null +++ b/sequencer-core/src/scheduler/fold.rs @@ -0,0 +1,583 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The recovery replay/fold engine: reconstruct logical app state and +//! the resume batch nonce from a trusted checkpoint plus an L1 input range. +//! +//! `(machine@checkpoint, L1 range) → (machine@S', resume nonce N)`. The engine +//! is a thin, deterministic driver over the [`Scheduler`] library: it seeds the +//! fridge from the `(A, B]` directs (step 3), replays the `(B, C]` stream +//! (step 4), drains the leftover fridge at `C` (step 5), and returns the +//! advanced [`Application`] state plus the scheduler's resume nonce. +//! +//! Read-only and pure: it never touches L1 or storage. The caller sources the +//! inputs (`setup --recovery` from the synced `safe_inputs`; a future standalone +//! tool from raw L1) and persists the result. Determinism is inherited from the +//! scheduler (FIFO fridge, fixed-width arithmetic, no iteration-order leakage); +//! the engine's only added obligation is that the caller feed inputs in +//! ascending L1 order within `(A, C]` (seeds `(A, B]` before replay `(B, C]`). +//! That contract is enforced fail-loud with always-on assertions — a violation +//! would silently drop a direct and yield a wrong recovered state. + +use alloy_primitives::Address; +use alloy_sol_types::Eip712Domain; + +use super::{Scheduler, SchedulerConfig, SchedulerInput}; +use crate::application::Application; + +/// One reconstructed L1 input for the fold, ordered ascending by inclusion +/// block (ties broken by safe-input index at the source). Mirrors +/// [`SchedulerInput`] minus the EIP-712 `domain` — that is one +/// deployment-wide constant the engine clones onto each input rather than +/// storing per row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FoldInput { + pub sender: Address, + pub inclusion_block: u64, + pub payload: Vec, +} + +/// Run recovery steps 3–5 over a trusted checkpoint and an L1 range, +/// returning `(S', N')` = (advanced app state, resume batch nonce). +/// +/// - `app` / `checkpoint_nonce`: the trusted checkpoint machine `S` at block +/// `B` and its scheduler nonce `N` (metadata — the bare-metal app cannot +/// recompute it, so the engine is *told* it via `resume_at`). +/// - `seeds`: directs reconstructed from `(A, B]`, with sequencer-sourced +/// batches already dropped by the caller (their content is already in `S`, +/// their frames' safe blocks `≤ A`). Must arrive in ascending L1 order. +/// - `replay`: the full `(B, C]` stream (batches + directs) in L1 order. The +/// scheduler classifies each input (batch iff `sender == sequencer_address`), +/// force-executes overdue directs on arrival, applies accepted batches, +/// drains covered fridge directs, and advances the nonce to `N'`. +/// - `stop_block`: `C`, the stable stopping block. The terminal drain runs here. +/// +/// `checkpoint_nonce` is trusted unchallenged (the operator's checkpoint is +/// trusted input; there is no cheap on-chain oracle to validate it against). +/// Notices/vouchers produced during the fold are dropped: +/// recovery reconstructs *logical* state (balances, nonces), not the L1-facing +/// output stream. +pub fn fold_replay( + app: A, + checkpoint_nonce: u64, + config: SchedulerConfig, + domain: Eip712Domain, + seeds: S, + replay: R, + stop_block: u64, +) -> (A, u64) +where + A: Application, + S: IntoIterator, + R: IntoIterator, +{ + let sequencer_address = config.sequencer_address; + let mut scheduler = Scheduler::resume_at(app, config, checkpoint_nonce); + + // The engine's input contract (ascending L1 order, seeds in (A, B] before + // replay in (B, C], everything `<= C`) is enforced fail-loud with always-on + // `assert!` rather than `debug_assert!`: a violation means the caller sourced + // the wrong inputs, and the failure mode is silent state divergence (a + // dropped direct → a wrong recovered `S'` that still *looks* valid). For a + // recovery engine that is the worst outcome, and the checks are negligible + // against a once-per-recovery fold, so we pay for them in release too. + + // Step 3 — reconstruct the fridge as of B from the (A, B] directs. + let mut last_seed_block: Option = None; + for seed in seeds { + assert!( + last_seed_block.is_none_or(|prev| seed.inclusion_block >= prev), + "fold seeds must arrive in ascending inclusion-block order" + ); + // SC-3: seeds are reconstructed `(A, B]` *directs* — enqueue_direct skips + // sender classification, so a mis-sourced `sender == sequencer_address` + // row would be silently filed as a direct instead of a batch (a wrong + // S'). Fail loud, matching the ordering/range asserts above. + assert!( + seed.sender != sequencer_address, + "fold seed must be a direct input, not a sequencer batch (sender == sequencer_address)" + ); + last_seed_block = Some(seed.inclusion_block); + scheduler.enqueue_direct(seed.sender, seed.inclusion_block, seed.payload); + } + + // Step 4 — replay (B, C] through the scheduler. + let mut last_replay_block: Option = None; + for input in replay { + if last_replay_block.is_none() { + // The first replay input opens `(B, C]`, which is strictly disjoint + // from the seeds' `(A, B]`: a seed block is `≤ B`, a replay block is + // `> B`, so the first replay block must come *strictly after* the + // last seeded direct. (A shared boundary block would let the FIFO + // fridge interleave the two ranges out of L1 order, since all seeds + // are enqueued before any replay input regardless of within-block + // index — see the disjoint-range contract in the docstring.) + assert!( + last_seed_block.is_none_or(|seed| input.inclusion_block > seed), + "fold replay must start strictly after the last seeded direct (ranges are disjoint)" + ); + } + assert!( + last_replay_block.is_none_or(|prev| input.inclusion_block >= prev), + "fold replay inputs must arrive in ascending inclusion-block order" + ); + last_replay_block = Some(input.inclusion_block); + let _ = scheduler.process_input(SchedulerInput { + sender: input.sender, + inclusion_block: input.inclusion_block, + domain: domain.clone(), + payload: input.payload, + }); + } + + // Step 5 — drain the leftover fridge at C. Every still-queued direct has + // `inclusion_block <= C`, so this executes them all; the booting run, which + // starts past C with no fridge, would otherwise never see them. + let _ = scheduler.drain_covered_at(stop_block); + + // Fail loud on a dropped direct: after draining at C the fridge MUST be + // empty. A non-empty fridge means an input arrived with `inclusion_block > C` + // (seed or replay) — outside the fold's range — which `finish` would + // otherwise discard silently, yielding a wrong `S'`. + assert_eq!( + scheduler.queued_direct_len(), + 0, + "fold left {} direct(s) undrained at stop block {stop_block}: an input \ + had inclusion_block > C (caller sourced inputs outside (A, C])", + scheduler.queued_direct_len(), + ); + + scheduler.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::application::{AppError, AppOutputs, InvalidReason}; + use crate::batch::{Batch, Frame}; + use crate::l2_tx::{DirectInput, ValidUserOp}; + use crate::user_op::UserOp; + use alloy_primitives::address; + + const SEQUENCER: Address = address!("0x1111111111111111111111111111111111111111"); + const DIRECT_SENDER: Address = address!("0x2222222222222222222222222222222222222222"); + const MAX_WAIT: u64 = 100; + + fn config() -> SchedulerConfig { + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks: MAX_WAIT, + } + } + + fn domain() -> Eip712Domain { + super::super::input_domain(1, Address::ZERO) + } + + /// Minimal `Application` double for the engine tests: records the markers of + /// executed direct inputs (first payload byte) in order, and advances a + /// safe-block clock. No user ops are exercised here — the engine drives the + /// scheduler over directs and empty/cover-only batches, so the user-op path + /// is trivial. Snapshot lifecycle is unused by the fold. + #[derive(Default, Clone)] + struct FoldApp { + executed_directs: Vec, + safe_block: u64, + } + + impl Application for FoldApp { + const MAX_METHOD_PAYLOAD_BYTES: usize = 1 + 32 + 20; + + fn validate_user_op( + &self, + _sender: Address, + _user_op: &UserOp, + _current_fee: u16, + ) -> Result<(), InvalidReason> { + Ok(()) + } + + fn execute_valid_user_op( + &mut self, + _user_op: &ValidUserOp, + safe_block: u64, + ) -> Result { + self.safe_block = self.safe_block.max(safe_block); + Ok(Vec::new()) + } + + fn execute_direct_input(&mut self, input: &DirectInput) -> Result { + self.executed_directs + .push(input.payload.first().copied().unwrap_or(0)); + self.safe_block = self.safe_block.max(input.block_number); + Ok(Vec::new()) + } + + fn executed_input_count(&self) -> u64 { + self.executed_directs.len() as u64 + } + + fn last_executed_safe_block(&self) -> u64 { + self.safe_block + } + + fn from_dump(_prefix: &std::path::Path) -> Result { + unimplemented!("FoldApp does not participate in snapshot lifecycle") + } + fn create_dump(&self, _prefix: &std::path::Path) -> Result<(), AppError> { + unimplemented!("FoldApp does not participate in snapshot lifecycle") + } + fn delete_dump(_prefix: &std::path::Path) -> Result<(), AppError> { + unimplemented!("FoldApp does not participate in snapshot lifecycle") + } + fn state_file_in_dump(_prefix: &std::path::Path) -> std::path::PathBuf { + unimplemented!("FoldApp does not participate in snapshot lifecycle") + } + } + + fn direct(sender: Address, block: u64, marker: u8) -> FoldInput { + FoldInput { + sender, + inclusion_block: block, + payload: vec![marker], + } + } + + /// A batch input (from the sequencer) carrying `batch`, SSZ-encoded. + fn batch(block: u64, batch: Batch) -> FoldInput { + FoldInput { + sender: SEQUENCER, + inclusion_block: block, + payload: ssz::Encode::as_ssz_bytes(&batch), + } + } + + /// An empty batch (no frames) at `nonce` — a scheduler no-op that only + /// advances the expected nonce. + fn empty_batch(block: u64, nonce: u64) -> FoldInput { + batch( + block, + Batch { + nonce, + frames: vec![], + }, + ) + } + + /// A batch at `nonce` with one user-op-free frame at `safe_block` — covers + /// (drains) fridge directs with `inclusion_block <= safe_block` without + /// needing signed user ops. + fn cover_batch(block: u64, nonce: u64, safe_block: u64) -> FoldInput { + batch( + block, + Batch { + nonce, + frames: vec![Frame { + user_ops: vec![], + safe_block, + fee_price: 0, + }], + }, + ) + } + + #[test] + fn empty_fold_returns_checkpoint_state_and_nonce() { + let (app, n) = fold_replay( + FoldApp::default(), + 7, + config(), + domain(), + Vec::new(), + Vec::new(), + 1_000, + ); + assert_eq!(n, 7, "no batches ⇒ nonce stays at the checkpoint"); + assert!(app.executed_directs.is_empty()); + } + + #[test] + fn resume_nonce_advances_from_checkpoint_through_replayed_batches() { + // Start at N0 = 5 (proves resume_at is honored, not 0); replay three + // empty batches at the expected nonces ⇒ N' = 8. + let replay = vec![ + empty_batch(100, 5), + empty_batch(110, 6), + empty_batch(120, 7), + ]; + let (_app, n) = fold_replay( + FoldApp::default(), + 5, + config(), + domain(), + Vec::new(), + replay, + 1_000, + ); + assert_eq!(n, 8, "three accepted batches advance N from 5 to 8"); + } + + #[test] + fn seeded_fridge_directs_drain_on_a_covering_batch_frame() { + // Seed two (A,B] directs; replay a batch whose frame safe_block covers + // both ⇒ they execute in FIFO order, before the nonce advances. + let seeds = vec![ + direct(DIRECT_SENDER, 10, 0xAA), + direct(DIRECT_SENDER, 20, 0xBB), + ]; + let replay = vec![cover_batch(30, 0, 20)]; + let (app, n) = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds, + replay, + 1_000, + ); + assert_eq!( + app.executed_directs, + vec![0xAA, 0xBB], + "directs drain in L1 order" + ); + assert_eq!(n, 1, "the covering batch advances the nonce"); + } + + #[test] + fn step5_drains_all_leftover_directs_even_when_not_overdue() { + // A direct seeded at block 25, no covering batch, C = 30, MAX_WAIT = 100. + // The direct is NOT overdue at C (25 + 100 > 30), but it IS covered by C, + // so step 5 must still execute it — proving drain-covered-at-C, not + // drain-overdue. (An overdue-only drain would lose it.) + let seeds = vec![direct(DIRECT_SENDER, 25, 0xCC)]; + let (app, _n) = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds, + Vec::new(), + 30, + ); + assert_eq!( + app.executed_directs, + vec![0xCC], + "a non-overdue leftover direct ≤ C must be drained at step 5" + ); + } + + #[test] + fn fold_is_deterministic_over_identical_inputs() { + let seeds = vec![ + direct(DIRECT_SENDER, 5, 0x01), + direct(DIRECT_SENDER, 15, 0x02), + ]; + let replay = vec![cover_batch(40, 0, 15), empty_batch(50, 1)]; + let run = || { + fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds.clone(), + replay.clone(), + 100, + ) + }; + let (app_a, n_a) = run(); + let (app_b, n_b) = run(); + assert_eq!(app_a.executed_directs, app_b.executed_directs); + assert_eq!(app_a.safe_block, app_b.safe_block); + assert_eq!(n_a, n_b); + } + + // --- fail-loud contract guards (always-on, not debug-only) --- + + #[test] + #[should_panic(expected = "undrained at stop block")] + fn replay_input_past_stop_block_panics_instead_of_silently_dropping() { + // A direct at block 50 with C = 30: never covered, never overdue, so it + // lingers in the fridge after step 5. The booting run would never see it, + // so a silent drop would diverge `S'` — the guard must catch it. + let replay = vec![direct(DIRECT_SENDER, 50, 0xDD)]; + let _ = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + Vec::new(), + replay, + 30, + ); + } + + #[test] + #[should_panic(expected = "seeds must arrive in ascending")] + fn out_of_order_seeds_panic() { + let seeds = vec![ + direct(DIRECT_SENDER, 20, 0x01), + direct(DIRECT_SENDER, 10, 0x02), + ]; + let _ = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds, + Vec::new(), + 100, + ); + } + + #[test] + #[should_panic(expected = "replay inputs must arrive in ascending")] + fn out_of_order_replay_panics() { + let replay = vec![ + direct(DIRECT_SENDER, 100, 0x01), + direct(DIRECT_SENDER, 50, 0x02), + ]; + let _ = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + Vec::new(), + replay, + 200, + ); + } + + #[test] + #[should_panic(expected = "replay must start strictly after")] + fn replay_starting_before_last_seed_panics() { + // Seed a direct at B-ish (block 50); a replay input at block 10 precedes + // it, which would interleave the (A,B] and (B,C] ranges out of L1 order. + let seeds = vec![direct(DIRECT_SENDER, 50, 0x01)]; + let replay = vec![direct(DIRECT_SENDER, 10, 0x02)]; + let _ = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds, + replay, + 100, + ); + } + + #[test] + #[should_panic(expected = "replay must start strictly after")] + fn replay_sharing_the_last_seed_block_panics() { + // The (A,B] seeds and (B,C] replay are strictly disjoint: a replay input + // at the SAME block as the last seed violates the contract (and would + // risk FIFO interleaving), so the boundary assert is strict (`>`). + let seeds = vec![direct(DIRECT_SENDER, 30, 0x01)]; + let replay = vec![direct(DIRECT_SENDER, 30, 0x02)]; + let _ = fold_replay( + FoldApp::default(), + 0, + config(), + domain(), + seeds, + replay, + 100, + ); + } + + #[test] + fn fold_replay_reconstructs_identical_state_to_a_live_run() { + // Differential: splitting the (genesis, C] stream at an intermediate B + // and running fold_replay over (checkpoint@B, (A,B] seeds, (B,C] replay) + // must reconstruct EXACTLY the (state, nonce) a single uninterrupted live + // run produces. This is the property the (C,H1] bug violated — and + // idempotence (the existing `fold_is_deterministic` test) cannot catch a + // systematic reconstruction error, because it only re-runs fold against + // itself. + + // Full input stream over (genesis, C=30], deliberately shaped so block B + // = 15 leaves a non-empty (A,B] fridge (the seed path): + // block 5 direct(1) — executed pre-A by the batch@10 + // block 10 batch nonce0 sb5 — drains direct(1); A := 5 + // block 12 direct(2) — UNDRAINED at B=15 ⇒ a seed in (A=5, B=15] + // block 20 batch nonce1 sb12 — drains the seed direct(2) + // block 25 direct(3) — drained only by the terminal drain at C + let covering_batch = |nonce: u64, safe_block: u64| { + batch( + if nonce == 0 { 10 } else { 20 }, + Batch { + nonce, + frames: vec![Frame { + user_ops: vec![], + safe_block, + fee_price: 0, + }], + }, + ) + }; + let pre_b = || { + vec![ + direct(DIRECT_SENDER, 5, 1), + covering_batch(0, 5), + direct(DIRECT_SENDER, 12, 2), + ] + }; + let post_b = || vec![covering_batch(1, 12), direct(DIRECT_SENDER, 25, 3)]; + let stop = 30; + + let feed = |scheduler: &mut Scheduler, inputs: Vec| { + for input in inputs { + scheduler.process_input(SchedulerInput { + sender: input.sender, + inclusion_block: input.inclusion_block, + domain: domain(), + payload: input.payload, + }); + } + }; + + // LIVE: one scheduler over the whole stream, terminal-drained at C. + let mut live = Scheduler::new(FoldApp::default(), config()); + feed(&mut live, pre_b()); + feed(&mut live, post_b()); + live.drain_covered_at(stop); + let (live_app, live_nonce) = live.finish(); + + // CHECKPOINT @ B: a scheduler over (genesis, B] then `finish` WITHOUT a + // drain — the undrained (A,B] directs are dropped from S and re-supplied + // as seeds. This is exactly the checkpoint contract: every batch ≤ B + // applied, no (A,B] direct executed yet. + let mut at_b = Scheduler::new(FoldApp::default(), config()); + feed(&mut at_b, pre_b()); + let (checkpoint_app, checkpoint_nonce) = at_b.finish(); + let a = checkpoint_app.last_executed_safe_block(); + assert!(a < 12 && 12 <= 15, "the seed direct(2) must sit in (A, B]"); + + // RECOVERY: fold the checkpoint over the (A,B] seeds + (B,C] replay to C. + let seeds = vec![direct(DIRECT_SENDER, 12, 2)]; + let (recovered_app, recovered_nonce) = fold_replay( + checkpoint_app, + checkpoint_nonce, + config(), + domain(), + seeds, + post_b(), + stop, + ); + + assert_eq!( + recovered_nonce, live_nonce, + "resume nonce N' must equal the live run's nonce" + ); + assert_eq!( + recovered_app.executed_directs, live_app.executed_directs, + "executed directs (content AND order) must equal the live run" + ); + assert_eq!( + recovered_app.last_executed_safe_block(), + live_app.last_executed_safe_block(), + "the safe-block clock must equal the live run" + ); + // Sanity: the stream actually executed all three directs (not a trivial pass). + assert_eq!(recovered_app.executed_directs, vec![1, 2, 3]); + } +} diff --git a/examples/canonical-app/src/scheduler/core.rs b/sequencer-core/src/scheduler/mod.rs similarity index 63% rename from examples/canonical-app/src/scheduler/core.rs rename to sequencer-core/src/scheduler/mod.rs index 07c641f..4f1aa00 100644 --- a/examples/canonical-app/src/scheduler/core.rs +++ b/sequencer-core/src/scheduler/mod.rs @@ -1,50 +1,44 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use alloy_primitives::{Address, Signature, U256, address}; +pub mod fold; + +pub use fold::{FoldInput, fold_replay}; + +use crate::application::{AppOutputs, Application}; +use crate::batch::{Batch, Frame, WireUserOp}; +use crate::l2_tx::DirectInput; +use alloy_primitives::{Address, Signature}; use alloy_sol_types::Eip712Domain; use alloy_sol_types::SolStruct; -use sequencer_core::application::{AppOutputs, Application}; -use sequencer_core::batch::{Batch, Frame, WireUserOp}; -use sequencer_core::l2_tx::DirectInput; use std::collections::VecDeque; -pub const DEVNET_SEQUENCER_ADDRESS: Address = - address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); -pub const SEPOLIA_SEQUENCER_ADDRESS: Address = - address!("0x16d5FF3Fdd14e2a86FBA77cbcE6B3Cd9C32b8Ff3"); -pub const MAX_WAIT_BLOCKS: u64 = sequencer_core::MAX_WAIT_BLOCKS; +pub const MAX_WAIT_BLOCKS: u64 = crate::MAX_WAIT_BLOCKS; #[derive(Debug, Clone, PartialEq, Eq)] pub struct SchedulerConfig { + /// L1 address whose inputs are trusted as sequencer batches; every other + /// sender is a direct input. This is deployment/app data (which key the + /// sequencer submits batches from), supplied by the app binary — the + /// protocol library never hardcodes a concrete sequencer address. pub sequencer_address: Address, pub max_wait_blocks: u64, } impl SchedulerConfig { - pub const fn devnet() -> Self { - Self { - sequencer_address: DEVNET_SEQUENCER_ADDRESS, - max_wait_blocks: MAX_WAIT_BLOCKS, - } - } - - pub const fn sepolia() -> Self { + /// Production config for `sequencer_address`, pinning the staleness window + /// to the protocol's [`MAX_WAIT_BLOCKS`]. Tests that need a custom window + /// construct the struct directly. + pub const fn new(sequencer_address: Address) -> Self { Self { - sequencer_address: SEPOLIA_SEQUENCER_ADDRESS, + sequencer_address, max_wait_blocks: MAX_WAIT_BLOCKS, } } } -impl Default for SchedulerConfig { - fn default() -> Self { - Self::sepolia() - } -} - #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct SchedulerInput { +pub struct SchedulerInput { pub sender: Address, pub inclusion_block: u64, pub domain: Eip712Domain, @@ -52,7 +46,7 @@ pub(super) struct SchedulerInput { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum ProcessOutcome { +pub enum ProcessOutcome { DirectEnqueued, BatchExecuted, BatchSkippedStale, @@ -60,7 +54,7 @@ pub(super) enum ProcessOutcome { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct ProcessResult { +pub struct ProcessResult { pub outcome: ProcessOutcome, pub outputs: AppOutputs, } @@ -91,13 +85,13 @@ impl PartialEq for ProcessOutcome { pub const STATE_INSPECT_QUERY: &[u8] = b"state"; #[derive(Debug, PartialEq, Eq)] -pub(super) enum InspectError { +pub enum InspectError { UnsupportedQuery, Application(String), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum BatchRejectReason { +pub enum BatchRejectReason { DecodeFailed, WrongNonce { expected: u64, got: u64 }, SafeBlockAboveInclusionBlock, @@ -120,26 +114,75 @@ struct QueuedDirectInput { } impl Scheduler { - pub fn new(app: A, config: SchedulerConfig) -> Self { + /// Construct a scheduler that begins expecting batch nonce + /// `next_expected_batch_nonce`. The genesis scheduler starts at 0 + /// ([`Scheduler::new`]); the recovery fold engine resumes at the + /// checkpoint's batch nonce, which the bare-metal app cannot recompute — it + /// rides as checkpoint metadata. + /// Sets the nonce field directly (not via the private advance). + pub fn resume_at(app: A, config: SchedulerConfig, next_expected_batch_nonce: u64) -> Self { Self { app, config, direct_q: VecDeque::new(), - next_expected_batch_nonce: 0, + next_expected_batch_nonce, } } - #[cfg(test)] + pub fn new(app: A, config: SchedulerConfig) -> Self { + Self::resume_at(app, config, 0) + } + + /// Number of directs still queued in the fridge. The recovery fold asserts + /// this is zero after draining at the stop block `C`: any leftover direct + /// means an input arrived with `inclusion_block > C` (a caller contract + /// violation), which would otherwise be silently dropped by [`finish`]. pub fn queued_direct_len(&self) -> usize { self.direct_q.len() } - #[cfg(test)] + /// The batch nonce the scheduler next expects (and that a resumed sequencer + /// submits at). The recovery fold reads it via [`Scheduler::finish`] as the + /// resume nonce `N`. pub fn next_expected_batch_nonce(&self) -> u64 { self.next_expected_batch_nonce } - pub(super) fn inspect_state(&self, query: &[u8]) -> Result, InspectError> { + /// Seed the fridge (direct-input queue) with a reconstructed direct. The + /// recovery fold uses this to rebuild the scheduler's `(A, B]` fridge before + /// replaying `(B, C]`. Callers MUST enqueue in ascending + /// L1 order (`inclusion_block`) so the FIFO drain matches the on-chain order. + pub fn enqueue_direct(&mut self, sender: Address, inclusion_block: u64, payload: Vec) { + self.direct_q.push_back(QueuedDirectInput { + sender, + payload, + inclusion_block, + }); + } + + /// Execute every fridge direct covered by `safe_block` (those with + /// `inclusion_block <= safe_block`), returning their outputs. The recovery + /// fold calls this once at the stopping block `C`: every + /// still-queued direct has `inclusion_block <= C`, so this drains them all — + /// exactly what the booting run's first frame at a safe block `>= C` would + /// do, except the booting run (bare-metal, no fridge) never sees them. + pub fn drain_covered_at(&mut self, safe_block: u64) -> AppOutputs { + let mut outputs = Vec::new(); + self.drain_directs_safe_at(safe_block, &mut outputs); + outputs + } + + /// Consume the scheduler, returning the advanced application state `S'` and + /// the resume batch nonce `N`: `S'` is the folded app state, `N` + /// the nonce the new sequencer resumes submitting at. + pub fn finish(self) -> (A, u64) { + (self.app, self.next_expected_batch_nonce) + } + + /// Watchdog / CM `inspect_state` hook: the app's canonical snapshot bytes + /// for the `/finalized_state` byte-compare. `pub` (not `pub(super)`) because + /// the canonical-app harness is now a separate crate over this library. + pub fn inspect_state(&self, query: &[u8]) -> Result, InspectError> { if !query.is_empty() && query != STATE_INSPECT_QUERY { return Err(InspectError::UnsupportedQuery); } @@ -149,7 +192,7 @@ impl Scheduler { .map_err(|err| InspectError::Application(err.to_string())) } - pub(super) fn process_input(&mut self, input: SchedulerInput) -> ProcessResult { + pub fn process_input(&mut self, input: SchedulerInput) -> ProcessResult { // Execute overdue directs before any input to keep backstop semantics explicit. let mut outputs = Vec::new(); self.force_execute_overdue(input.inclusion_block, &mut outputs); @@ -253,7 +296,7 @@ impl Scheduler { /// Execute user-ops in a frame, skipping any whose `max_fee` is below the frame's `fee_price`. /// /// Both `max_fee` and `fee_price` are log-space exponents (base 129/128). - /// See [`sequencer_core::fee`] for conversion to linear amounts. + /// See [`crate::fee`] for conversion to linear amounts. fn execute_frame_user_ops( &mut self, domain: &Eip712Domain, @@ -261,28 +304,36 @@ impl Scheduler { outputs: &mut AppOutputs, ) { for user_op in &frame.user_ops { + // An unrecoverable signature is dropped silently (the scheduler is a + // pure deterministic fold; diagnostics would be a nondeterministic + // side effect at the library seam). if let Some(sender) = self.recover_sender(domain, user_op) { let plain = user_op.to_user_op(); - // Defense-in-depth: the trait default in validate_and_execute_user_op - // now centralizes this check, but we keep it here as an extra guard. - if plain.max_fee < frame.fee_price { - eprintln!("scheduler skipped frame user-op due to max_fee < fee_price"); - continue; - } - match self - .app - .validate_and_execute_user_op(sender, &plain, frame.fee_price) - { - Ok(sequencer_core::application::ExecutionOutcome::Included { + match crate::application::validate_and_execute_user_op( + &mut self.app, + sender, + &plain, + frame.fee_price, + frame.safe_block, + ) { + Ok(crate::application::ExecutionOutcome::Included { outputs: user_op_outputs, }) => outputs.extend(user_op_outputs), - Ok(sequencer_core::application::ExecutionOutcome::Invalid(_)) => {} - Err(err) => { - eprintln!("scheduler skipped frame user-op due to app error: {err}"); - } + // Invalid op or app error: skip it (no state change, no output). + // + // The `Err(AppError)` arm is the canonical (fold) half of an + // asymmetry with the inclusion lane (`execute_user_op` in + // `inclusion_lane/mod.rs`), which fails *loud* on the same + // error. Duality (I1) still holds: an `Err` excludes the op + // from state on *both* sides (neither extends `outputs`), so + // the canonical state agrees — the lane merely additionally + // aborts, treating the error as the internal-invariant breach + // it is. Dead by construction today: the wallet app's + // `execute_valid_user_op` errors only on a fee/balance check + // it already passed in `validate_user_op`, which cannot change + // between the two calls within one fold step. + Ok(crate::application::ExecutionOutcome::Invalid(_)) | Err(_) => {} } - } else { - eprintln!("scheduler skipped frame user-op due to invalid signature"); } } } @@ -308,9 +359,9 @@ impl Scheduler { block_number: queued.inclusion_block, payload: queued.payload, }; - match self.app.execute_direct_input(&input) { - Ok(direct_outputs) => outputs.extend(direct_outputs), - Err(err) => eprintln!("scheduler failed to execute drained direct input: {err}"), + // A failing direct is skipped (deterministic fold; no diagnostics). + if let Ok(direct_outputs) = self.app.execute_direct_input(&input) { + outputs.extend(direct_outputs); } } } @@ -327,11 +378,9 @@ impl Scheduler { block_number: front.inclusion_block, payload: front.payload.clone(), }; - match self.app.execute_direct_input(&input) { - Ok(direct_outputs) => outputs.extend(direct_outputs), - Err(err) => { - eprintln!("scheduler failed to execute overdue direct input: {err}") - } + // A failing overdue direct is skipped (deterministic fold). + if let Ok(direct_outputs) = self.app.execute_direct_input(&input) { + outputs.extend(direct_outputs); } self.direct_q.pop_front().expect("queue front must exist"); @@ -346,27 +395,17 @@ fn has_elapsed_since(start_block: u64, wait_blocks: u64, current_block: u64) -> current_block.saturating_sub(start_block) >= wait_blocks } -pub(super) fn input_domain(chain_id: u64, verifying_contract: Address) -> Eip712Domain { - sequencer_core::build_input_domain(chain_id, verifying_contract) -} - -pub(super) fn block_to_u64(block: U256) -> u64 { - // Solidity ABI exposes block numbers as uint256, but scheduler semantics use u64. - // A value that does not fit u64 is a malformed host input for this prototype. - u64::try_from(block).expect("block number does not fit u64") -} - -pub(super) fn chain_id_to_u64(chain_id: U256) -> u64 { - u64::try_from(chain_id).expect("chain id does not fit u64") +pub fn input_domain(chain_id: u64, verifying_contract: Address) -> Eip712Domain { + crate::build_input_domain(chain_id, verifying_contract) } #[cfg(test)] mod tests { use super::*; - use alloy_primitives::address; + use crate::user_op::UserOp; + use alloy_primitives::{U256, address}; use k256::ecdsa::SigningKey; use k256::ecdsa::signature::hazmat::PrehashSigner; - use sequencer_core::user_op::UserOp; #[cfg(test)] #[derive(Default)] @@ -374,6 +413,7 @@ mod tests { executed: Vec, balances: std::collections::HashMap, nonces: std::collections::HashMap, + last_executed_safe_block: u64, } #[cfg(test)] @@ -406,58 +446,52 @@ mod tests { #[cfg(test)] impl Application for RecordingApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = app_core::application::MAX_METHOD_PAYLOAD_BYTES; - - fn current_user_nonce(&self, _sender: Address) -> u32 { - self.nonce_of(_sender) - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - self.balance_of(_sender) - } + // Mirrors the wallet app's method-payload cap (selector + amount + + // address). A local literal keeps sequencer-core free of an app-core + // dependency (which would invert the crate graph). + const MAX_METHOD_PAYLOAD_BYTES: usize = 1 + 32 + 20; fn validate_user_op( &self, sender: Address, - user_op: &sequencer_core::user_op::UserOp, + user_op: &crate::user_op::UserOp, current_fee: u16, - ) -> Result<(), sequencer_core::application::InvalidReason> { + ) -> Result<(), crate::application::InvalidReason> { let expected_nonce = self.nonce_of(sender); if user_op.nonce != expected_nonce { - return Err(sequencer_core::application::InvalidReason::InvalidNonce { + return Err(crate::application::InvalidReason::InvalidNonce { expected: expected_nonce, got: user_op.nonce, }); } if user_op.max_fee < current_fee { - return Err(sequencer_core::application::InvalidReason::InvalidMaxFee { + return Err(crate::application::InvalidReason::InvalidMaxFee { max_fee: user_op.max_fee, base_fee: current_fee, }); } - let required = sequencer_core::fee::fee_to_linear(current_fee); + let required = crate::fee::fee_to_linear(current_fee); let balance = self.balance_of(sender); if balance < required { - return Err( - sequencer_core::application::InvalidReason::InsufficientGasBalance { - required, - available: balance, - }, - ); + return Err(crate::application::InvalidReason::InsufficientFeeBalance { + required, + available: balance, + }); } Ok(()) } fn execute_valid_user_op( &mut self, - user_op: &sequencer_core::l2_tx::ValidUserOp, - ) -> Result - { + user_op: &crate::l2_tx::ValidUserOp, + safe_block: u64, + ) -> Result { + self.last_executed_safe_block = self.last_executed_safe_block.max(safe_block); let sender = user_op.sender; - let fee = sequencer_core::fee::fee_to_linear(user_op.fee); + let fee = crate::fee::fee_to_linear(user_op.fee); let balance = self.balance_of(sender); if balance < fee { - return Err(sequencer_core::application::AppError::Internal { + return Err(crate::application::AppError::Internal { reason: "validated user op cannot pay fee".to_string(), }); } @@ -473,29 +507,33 @@ mod tests { fn execute_direct_input( &mut self, input: &DirectInput, - ) -> Result - { + ) -> Result { let marker = input.payload.first().copied().unwrap_or(0); self.executed.push(RecordedTx::Direct(marker)); + self.last_executed_safe_block = self.last_executed_safe_block.max(input.block_number); Ok(Vec::new()) } - fn from_dump( - _prefix: &std::path::Path, - ) -> Result { + fn executed_input_count(&self) -> u64 { + self.executed.len() as u64 + } + + fn last_executed_safe_block(&self) -> u64 { + self.last_executed_safe_block + } + + fn from_dump(_prefix: &std::path::Path) -> Result { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } fn create_dump( &self, _prefix: &std::path::Path, - ) -> Result<(), sequencer_core::application::AppError> { + ) -> Result<(), crate::application::AppError> { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } - fn delete_dump( - _prefix: &std::path::Path, - ) -> Result<(), sequencer_core::application::AppError> { + fn delete_dump(_prefix: &std::path::Path) -> Result<(), crate::application::AppError> { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } @@ -503,9 +541,7 @@ mod tests { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } - fn canonical_snapshot_bytes( - &self, - ) -> Result, sequencer_core::application::AppError> { + fn canonical_snapshot_bytes(&self) -> Result, crate::application::AppError> { Ok(format!("events:{}", self.executed.len()).into_bytes()) } } @@ -1114,4 +1150,273 @@ mod tests { assert_eq!(scheduler.next_expected_batch_nonce(), 0); assert!(scheduler.app.events().is_empty()); } + + // ── I1 duality: off-chain predicate vs canonical fold ───────────────── + // + // `ProtocolTiming::scheduler_accepts` (the off-chain gold-frontier predicate) + // and `Scheduler::process_input` (the canonical fold) are hand-maintained in + // separate files and MUST agree on accept-vs-reject for every input — the + // system's most load-bearing invariant (I1). Nothing exercised both until + // now. The one *documented* divergence is the predicate's structural-reject + // omission (it trusts the sequencer to emit well-formed batches), pinned + // explicitly at the end so any *other* drift fails this test. + + const DUALITY_MAX_WAIT: u64 = 5; + + fn duality_timing() -> crate::protocol::ProtocolTiming { + crate::protocol::ProtocolTiming { + max_wait_blocks: DUALITY_MAX_WAIT, + preemptive_margin_blocks: 1, + l1_read_stale_after_blocks: 1, + seconds_per_block: 12, + } + } + + // A frame with no user ops — accept/reject is then purely about + // sender/nonce/structure/staleness, no app credit needed. + fn empty_frame(safe_block: u64) -> Frame { + Frame { + user_ops: vec![], + safe_block, + fee_price: 0, + } + } + + /// Run one input through BOTH sides at `expected_nonce`; return whether each + /// *accepted* it (canonical `BatchExecuted` / predicate `Some`). + fn duality_run( + sender: Address, + inclusion: u64, + expected_nonce: u64, + payload: &[u8], + ) -> (bool, bool) { + use crate::protocol::SafeInputView; + let mut scheduler = Scheduler::resume_at( + RecordingApp::default(), + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks: DUALITY_MAX_WAIT, + }, + expected_nonce, + ); + let canonical_executed = scheduler.process_input(SchedulerInput { + sender, + inclusion_block: inclusion, + domain: test_domain(), + payload: payload.to_vec(), + }) == ProcessOutcome::BatchExecuted; + let offchain_accepted = duality_timing() + .scheduler_accepts( + SEQUENCER, + SafeInputView { + safe_input_index: 0, + sender, + payload, + inclusion_block: inclusion, + }, + expected_nonce, + ) + .is_some(); + (canonical_executed, offchain_accepted) + } + + fn ssz(batch: &Batch) -> Vec { + ssz::Encode::as_ssz_bytes(batch) + } + + #[test] + fn scheduler_accepts_agrees_with_canonical_on_accept_reject() { + // (label, sender, inclusion, expected_nonce, payload) — structurally + // valid inputs where the two sides MUST agree. + let valid_fresh = ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(10)], + }); + let empty_frames = ssz(&Batch { + nonce: 0, + frames: vec![], + }); + let wrong_nonce = ssz(&Batch { + nonce: 1, + frames: vec![empty_frame(10)], + }); + let stale = ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(1)], + }); + let fresh_at_boundary = ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(10)], + }); + + let agree_cases: &[(&str, Address, u64, u64, &[u8])] = &[ + ("valid fresh, right nonce", SEQUENCER, 12, 0, &valid_fresh), + ( + "empty frames (no-op batch)", + SEQUENCER, + 100, + 0, + &empty_frames, + ), + ("wrong nonce", SEQUENCER, 12, 0, &wrong_nonce), + // age = 10 - 1 = 9 >= MAX_WAIT(5): stale on both sides. + ("stale by first frame", SEQUENCER, 10, 0, &stale), + // wrong sender: canonical enqueues a direct, predicate rejects sender. + ("wrong sender", DIRECT_SENDER, 12, 0, &valid_fresh), + // garbage payload: DecodeFailed / decode error. + ("garbage payload", SEQUENCER, 1, 0, &[0xFF, 0xEE, 0xDD]), + // staleness boundary: age = 14 - 10 = 4 < 5 accepted. + ( + "fresh just under stale", + SEQUENCER, + 14, + 0, + &fresh_at_boundary, + ), + // age = 15 - 10 = 5 >= 5: stale. + ( + "stale just at boundary", + SEQUENCER, + 15, + 0, + &fresh_at_boundary, + ), + ]; + + for (label, sender, inclusion, expected, payload) in agree_cases { + let (canonical, offchain) = duality_run(*sender, *inclusion, *expected, payload); + assert_eq!( + canonical, offchain, + "I1 disagreement on `{label}`: canonical_executed={canonical}, \ + offchain_accepted={offchain} — the predicate and the canonical \ + fold must agree on accept/reject" + ); + } + + // Documented exception — the predicate's structural-reject omission. The + // canonical fold rejects these (it checks frame structure); the predicate + // accepts them (self-trust: the sequencer never emits such batches). If + // either side changes, these asserts flip and force a deliberate update. + let non_monotonic = ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(8), empty_frame(7)], + }); + let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, &non_monotonic); + assert!( + !canonical && offchain, + "non-monotonic safe_blocks: expected canonical reject + predicate \ + accept (documented structural omission), got canonical={canonical} \ + offchain={offchain}" + ); + + let frame_above_inclusion = ssz(&Batch { + nonce: 0, + frames: vec![empty_frame(20)], + }); + let (canonical, offchain) = duality_run(SEQUENCER, 12, 0, &frame_above_inclusion); + assert!( + !canonical && offchain, + "frame safe_block > inclusion: expected canonical reject + predicate \ + accept (documented structural omission), got canonical={canonical} \ + offchain={offchain}" + ); + } + + #[test] + fn force_execute_overdue_runs_before_a_rejected_or_stale_batch() { + // The backstop force-executes overdue fridge directs at the START of + // process_input — before the batch is even classified. So an overdue + // direct is drained even when the same tick's batch is rejected or + // skipped (a danger-zone shape). Previously tested only alongside an + // ACCEPTED batch. + for label in ["wrong-nonce", "stale"] { + let mut scheduler = Scheduler::new( + RecordingApp::default(), + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks: 5, + }, + ); + // A direct at block 1; by inclusion block 8 it is overdue (age 7 >= 5). + scheduler.process_input(direct_input(1, 1)); + + let batch = if label == "wrong-nonce" { + // expected 0, got 9 → BatchRejected(WrongNonce). + Batch { + nonce: 9, + frames: vec![Frame { + user_ops: vec![], + safe_block: 8, + fee_price: 0, + }], + } + } else { + // right nonce but a stale first frame (age 8 - 1 = 7 >= 5). + Batch { + nonce: 0, + frames: vec![Frame { + user_ops: vec![], + safe_block: 1, + fee_price: 0, + }], + } + }; + + let outcome = scheduler.process_input(batch_input(8, batch)).outcome; + assert!( + matches!( + outcome, + ProcessOutcome::BatchRejected(_) | ProcessOutcome::BatchSkippedStale + ), + "{label}: batch should be rejected/skipped, got {outcome:?}" + ); + assert_eq!( + scheduler.app.events(), + [RecordedTx::Direct(1)], + "{label}: the overdue direct must be force-executed before the rejected batch" + ); + assert_eq!( + scheduler.next_expected_batch_nonce(), + 0, + "{label}: a rejected/stale batch consumes no nonce" + ); + } + } + + #[test] + fn multiframe_overheight_in_tail_frame_rejected_by_scheduler() { + // `batch_reject_reason_for_block` checks `safe_block <= inclusion` in + // BOTH the head and the tail frames; only the head case was tested. A + // tail frame above the inclusion block must reject the whole batch. + let mut scheduler = Scheduler::new( + RecordingApp::default(), + SchedulerConfig { + sequencer_address: SEQUENCER, + max_wait_blocks: 100, + }, + ); + let batch = Batch { + nonce: 0, + frames: vec![ + // head: 5 <= 10, valid. + Frame { + user_ops: vec![], + safe_block: 5, + fee_price: 0, + }, + // tail: 11 > 10 (the inclusion block) → reject. + Frame { + user_ops: vec![], + safe_block: 11, + fee_price: 0, + }, + ], + }; + assert_eq!( + scheduler.process_input(batch_input(10, batch)), + ProcessOutcome::BatchRejected(BatchRejectReason::SafeBlockAboveInclusionBlock) + ); + assert_eq!(scheduler.next_expected_batch_nonce(), 0); + assert!(scheduler.app.events().is_empty()); + } } diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 534eae0..3ae2f26 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -10,15 +10,14 @@ readme.workspace = true authors.workspace = true [dependencies] -app-core = { path = "../examples/app-core" } sequencer-core = { path = "../sequencer-core" } axum = { version = "0.8.8", features = ["ws"] } tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal", "fs", "io-util"] } tokio-util = { version = "0.7", features = ["io"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +toml = "0.8" tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } tower-http = { version = "0.6.8", features = ["trace"] } rusqlite = { version = "0.38.0", features = ["bundled"] } rusqlite_migration = "2.3.0" @@ -36,6 +35,7 @@ cartesi-rollups-contracts = "=2.2.0" async-trait = "0.1" [dev-dependencies] +app-core = { path = "../examples/app-core" } futures-util = "0.3" reqwest = { version = "0.12", default-features = false, features = ["stream"] } tokio-tungstenite = "0.28" diff --git a/sequencer/src/bin/sequencer-devnet.rs b/sequencer/src/bin/sequencer-devnet.rs deleted file mode 100644 index 676c19b..0000000 --- a/sequencer/src/bin/sequencer-devnet.rs +++ /dev/null @@ -1,19 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -use app_core::application::{WalletApp, WalletConfig}; -use clap::Parser; -use sequencer::{RunConfig, run}; - -#[tokio::main] -async fn main() -> Result<(), sequencer::RunError> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - - let config = RunConfig::parse(); - run(WalletApp::new(WalletConfig::devnet()), config).await -} diff --git a/sequencer/src/egress/l2_tx_feed/mod.rs b/sequencer/src/egress/l2_tx_feed/mod.rs index c7cf616..0b65a74 100644 --- a/sequencer/src/egress/l2_tx_feed/mod.rs +++ b/sequencer/src/egress/l2_tx_feed/mod.rs @@ -193,7 +193,10 @@ fn run_subscription( continue; } - for (db_offset, tx) in txs { + // The frame safe_block (third element) is a replay-only concern; + // the WS message shape doesn't carry it (review F7/WP5 owns any + // feed-protocol extension). + for (db_offset, tx, _frame_safe_block) in txs { if shutdown.is_shutdown_requested() || events_tx.is_closed() { return Ok(()); } diff --git a/sequencer/src/harness.rs b/sequencer/src/harness.rs new file mode 100644 index 0000000..f1b97a6 --- /dev/null +++ b/sequencer/src/harness.rs @@ -0,0 +1,97 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! CLI harness: the subcommand parser, dispatch, and R4 exit-code projection, +//! exported once from the library so every app binary inherits them. +//! +//! An app's `main` is ~5 lines: init tracing, then [`run_main`] with a +//! genesis-app factory closure. The factory is only invoked by `setup` (to +//! write the genesis finalized snapshot); `run` and `flush-mempool` never +//! construct an app value. +//! +//! ```ignore +//! #[tokio::main] +//! async fn main() -> std::process::ExitCode { +//! init_tracing(); +//! sequencer::harness::run_main(|| WalletApp::new(WalletConfig::default())).await +//! } +//! ``` +//! +//! Genesis construction stays off the `Application` trait (it varies per impl) +//! and is supplied by this closure. When a future app needs setup-time CLI +//! args of its own (e.g. a machine-image path), the extension point is a +//! `Cli` generic on the parser — deferred until an app needs it, so +//! the harness imposes no `clap` bound on the (possibly FFI) app type. + +use clap::{Parser, Subcommand}; +use sequencer_core::application::Application; + +use crate::runtime::config::{FlushConfig, RunConfig, SetupConfig}; + +/// Top-level CLI. Apps parse this (via [`run_main`]) and dispatch. +#[derive(Debug, Parser)] +#[command( + name = "sequencer", + version, + about = "App-specific rollup sequencer.\n\n\ + Subcommands: `setup` (pin identity + initial sync, run once), \ + `run` (boot the sequencer from a set-up DB), and `flush-mempool` \ + (settle the batch-submitter wallet nonce on demand).\n\n\ + All options can also be set via environment variables (shown in brackets)." +)] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +/// The three subcommands. Configs are boxed to keep the enum small (clippy +/// `large_enum_variant`); `RunConfig` is the largest. +#[derive(Debug, Subcommand)] +pub enum Command { + /// Establish the deployment's timeless state: pin identity, do the initial + /// L1 sync, register the genesis snapshot, and mark setup complete. Run + /// once before `run`. L1-read-only (takes the submitter address, not the + /// key). + Setup(Box), + /// Boot the sequencer from an already-set-up DB. Refuses unless `setup` + /// completed. Reads identity from the DB; keeps the signing key. + Run(Box), + /// Settle the batch-submitter wallet nonce on demand (operator tool). + FlushMempool(Box), +} + +/// Parse argv and dispatch. Returns the R4 process exit code. +pub async fn run_main(genesis_app: F) -> std::process::ExitCode +where + A: Application + Clone + Sync + 'static, + F: FnOnce() -> A, +{ + let cli = Cli::parse(); + dispatch(cli.command, genesis_app).await +} + +/// Dispatch a parsed [`Command`], projecting the result onto the R4 exit-code +/// contract (see [`crate::runtime::error`]). Clean completion is exit 0; every +/// `RunError` maps through `RunError::exit_code`. +/// +/// `genesis_app` is called at most once — only by `setup`. +pub async fn dispatch(command: Command, genesis_app: F) -> std::process::ExitCode +where + A: Application + Clone + Sync + 'static, + F: FnOnce() -> A, +{ + let result = match command { + Command::Setup(config) => crate::runtime::setup::setup(*config, genesis_app()).await, + Command::Run(config) => crate::runtime::run::(*config).await, + Command::FlushMempool(config) => crate::runtime::flush::flush_mempool(*config).await, + }; + + match result { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(err) => { + let code = err.exit_code(); + tracing::error!(error = %err, exit_code = code, "sequencer exiting"); + std::process::ExitCode::from(code) + } + } +} diff --git a/sequencer/src/ingress/inclusion_lane/catch_up.rs b/sequencer/src/ingress/inclusion_lane/catch_up.rs index b8ed53d..62a22a3 100644 --- a/sequencer/src/ingress/inclusion_lane/catch_up.rs +++ b/sequencer/src/ingress/inclusion_lane/catch_up.rs @@ -19,12 +19,13 @@ const DEFAULT_CATCH_UP_PAGE_SIZE: usize = 256; /// The single checkpoint the lane resumes from: the latest pending /// snapshot if any, else the finalized snapshot. Carries both the dump -/// `prefix` (to load the Application via `from_dump`) and the matching -/// `l2_tx_index` (the replay cursor), so the loaded state and the -/// catch-up cursor are guaranteed to come from the *same* checkpoint. +/// directory (whose `state` subtree loads the Application via +/// `from_dump`) and the matching `l2_tx_index` (the replay cursor), so +/// the loaded state and the catch-up cursor are guaranteed to come from +/// the *same* checkpoint. #[derive(Debug, Clone)] pub(super) struct CatchUpSnapshot { - pub(super) prefix: PathBuf, + pub(super) dump_dir: PathBuf, pub(super) l2_tx_index: u64, } @@ -35,18 +36,19 @@ pub(super) struct CatchUpSnapshot { /// back to the finalized snapshot. /// /// Returns the checkpoint directly rather than an `Option`: the -/// always-load invariant — `ensure_finalized_snapshot` runs in -/// `Workers::spawn` before `InclusionLane::start` — guarantees at least -/// the genesis finalized snapshot exists by the time the lane resumes. -/// Absence is a violated invariant (runtime/setup bug), surfaced -/// fail-loud as [`CatchUpError::NoSnapshot`]. +/// always-load invariant — `setup` registers the genesis finalized +/// snapshot, `run` gates on it (`require_finalized_snapshot` in +/// `Workers::spawn`, plus the boot-gate check) before `InclusionLane::start` +/// — guarantees at least the genesis finalized snapshot exists by the time +/// the lane resumes. Absence is a violated invariant (runtime/setup bug), +/// surfaced fail-loud as [`CatchUpError::NoSnapshot`]. pub(super) fn catch_up_snapshot(storage: &mut Storage) -> Result { let (dump, l2_tx_index) = storage .latest_snapshot() .map_err(|source| CatchUpError::LoadSnapshot { source })? .ok_or(CatchUpError::NoSnapshot)?; Ok(CatchUpSnapshot { - prefix: dump.prefix, + dump_dir: dump.prefix, l2_tx_index, }) } @@ -94,8 +96,8 @@ pub(super) fn catch_up_application_paged( return Ok(()); } - for (db_offset, item) in replay { - replay_sequenced_l2_tx(app, batch_submitter_address, item)?; + for (db_offset, item, frame_safe_block) in replay { + replay_sequenced_l2_tx(app, batch_submitter_address, item, frame_safe_block)?; next_offset = db_offset; } } @@ -105,10 +107,14 @@ fn replay_sequenced_l2_tx( app: &mut impl Application, batch_submitter_address: Address, item: SequencedL2Tx, + frame_safe_block: u64, ) -> Result<(), CatchUpError> { match item { SequencedL2Tx::UserOp(value) => { - app.execute_valid_user_op(&value) + // The persisted covering frame's safe_block mirrors what the + // lane passed live, so the replayed app's safe-block clock + // lands on the same value. + app.execute_valid_user_op(&value, frame_safe_block) .map(|_| ()) .map_err(|err| CatchUpError::ReplayUserOpInternal { reason: err.to_string(), diff --git a/sequencer/src/ingress/inclusion_lane/config.rs b/sequencer/src/ingress/inclusion_lane/config.rs index f0d2770..24f139b 100644 --- a/sequencer/src/ingress/inclusion_lane/config.rs +++ b/sequencer/src/ingress/inclusion_lane/config.rs @@ -55,4 +55,12 @@ impl InclusionLaneConfig { frontier_min_interval: DEFAULT_FRONTIER_MIN_INTERVAL, } } + + /// Override the force-close deadline (the operator-tunable + /// `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`). Builder so callers that only need the + /// defaults keep using [`InclusionLaneConfig::new`]. + pub fn with_max_batch_open(mut self, max_batch_open: Duration) -> Self { + self.max_batch_open = max_batch_open; + self + } } diff --git a/sequencer/src/ingress/inclusion_lane/dump_info.rs b/sequencer/src/ingress/inclusion_lane/dump_info.rs new file mode 100644 index 0000000..fd48ad9 --- /dev/null +++ b/sequencer/src/ingress/inclusion_lane/dump_info.rs @@ -0,0 +1,291 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Sequencer-owned dump-directory structure and metadata (`info.toml`). +//! +//! Every dump is a directory `dumps//` with exactly two entries: +//! +//! ```text +//! dumps// +//! state app-owned subtree — the prefix handed to +//! `Application::{create_dump, from_dump, delete_dump}` +//! info.toml sequencer-owned checkpoint metadata (this module) +//! ``` +//! +//! `info.toml` makes a finalized dump a self-contained checkpoint for the +//! recovery handoff — and, together with the app's `state` +//! subtree, the unit an operator backs up: `setup --recovery` rebuilds a wiped +//! DB from exactly this pair, reading `N` straight from the file. `next_batch_nonce` +//! (`N`) is known at batch close and written then; `promoted_inclusion_block` +//! (`B`) is known at promotion and stamped in place afterwards. An in-place update of a file +//! *inside* the dir changes no path, so the no-dangling-row invariant, leases, +//! and GC — all keyed on the immutable directory path — are untouched. +//! +//! The DB row (`dumps.prefix`) stores the dump *directory*; the app prefix is +//! always derived via [`app_prefix`]. Crash-safety ordering is unchanged from +//! the lifecycle doc: dir + `info.toml` + app dump are durable on disk before +//! the row that references the dir; row deletion precedes file deletion. + +use std::io; +use std::path::{Path, PathBuf}; + +use sequencer_core::application::{AppError, Application}; + +/// Name of the app-owned subtree inside a dump directory. +const APP_STATE_SUBDIR: &str = "state"; +/// Name of the sequencer-owned metadata file inside a dump directory. +const INFO_FILE: &str = "info.toml"; + +pub const FORMAT_VERSION: u64 = 1; + +/// The app's dump prefix inside `dump_dir`. Pure path derivation. +pub fn app_prefix(dump_dir: &Path) -> PathBuf { + dump_dir.join(APP_STATE_SUBDIR) +} + +/// Sequencer-owned checkpoint metadata for one dump. +/// +/// Serialized as real TOML (`#[serde(deny_unknown_fields)]` keeps the parse +/// strict — unknown keys, type mismatches, and TOML's own duplicate-key ban all +/// fail loud, the same guarantees the old hand parser gave). The on-disk layout +/// is plain `key = value` integer lines, so it is operator-readable and any +/// previously-written file round-trips unchanged. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DumpInfo { + pub format_version: u64, + /// `N` — the batch nonce the sequencer resumes submitting at when + /// booting from this checkpoint (snapshot batch's nonce + 1; 0 for + /// the genesis dump). Known at batch close. + pub next_batch_nonce: u64, + /// Replay cursor: global valid replay head at dump time. Mirrors the + /// snapshot row exactly. + pub l2_tx_index: u64, + /// `B` — the L1 inclusion block of the promotion that finalized this + /// dump. `None` until promotion; stamped in place when the batch is + /// observed accepted (and re-stamped from the DB at startup, closing + /// the commit-then-stamp crash window). Omitted from the file while + /// `None` (TOML has no null); a missing key reads back as `None`. + #[serde(skip_serializing_if = "Option::is_none")] + pub promoted_inclusion_block: Option, +} + +impl DumpInfo { + /// Checkpoint metadata for the snapshot taken at the close of batch + /// `batch_nonce`: the sequencer resumes submitting at `batch_nonce + 1`, and + /// `B` is unknown until promotion (stamped in place then). The single home + /// for the resume-nonce `+ 1` skew, so the batch-close and recovery-fill + /// sites cannot drift on how `next_batch_nonce` is derived. + pub fn at_batch_close(batch_nonce: u64, l2_tx_index: u64) -> Self { + Self { + format_version: FORMAT_VERSION, + next_batch_nonce: batch_nonce + 1, + l2_tx_index, + promoted_inclusion_block: None, + } + } + + /// Checkpoint metadata for a finalized snapshot rebuilt by `setup --recovery` + /// at inclusion block `inclusion_block`, resuming at `resume_nonce` — the + /// fold's next-expected nonce `N'`, already the resume value (no `+ 1`). + pub fn at_recovery(resume_nonce: u64, l2_tx_index: u64, inclusion_block: u64) -> Self { + Self { + format_version: FORMAT_VERSION, + next_batch_nonce: resume_nonce, + l2_tx_index, + promoted_inclusion_block: Some(inclusion_block), + } + } +} + +/// Errors from creating a structured dump directory. +#[derive(Debug, thiserror::Error)] +pub enum CreateDumpDirError { + #[error("app: {0}")] + App(#[from] AppError), + #[error("io: {0}")] + Io(#[from] io::Error), +} + +/// Create one structured dump directory: the dir itself, the +/// sequencer-owned `info.toml`, then the app's dump under `state` — +/// every file durable before the caller writes the DB row that +/// references the dir. The app's `create_dump` contract fsyncs its +/// subtree and its parent (the dump dir); the final parent-dir fsync +/// persists the dump dir's own entry in the dumps directory. +pub fn create_dump_dir_with_info( + app: &A, + dump_dir: &Path, + info: &DumpInfo, +) -> Result<(), CreateDumpDirError> { + std::fs::create_dir(dump_dir)?; + write_info(dump_dir, info)?; + app.create_dump(&app_prefix(dump_dir))?; + let parent = dump_dir + .parent() + .expect("dump dir always lives inside a dumps directory"); + std::fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +/// Delete one structured dump directory: the app's subtree via its +/// `delete_dump` hook (when present — an orphan from a crash between +/// dir creation and `create_dump` legitimately lacks it), then the +/// rest of the dir (`info.toml` + the dir itself). +pub fn delete_dump_dir(dump_dir: &Path) -> Result<(), AppError> { + let app_prefix = app_prefix(dump_dir); + if app_prefix.exists() { + A::delete_dump(&app_prefix)?; + } + std::fs::remove_dir_all(dump_dir)?; + Ok(()) +} + +/// Write `info.toml` into `dump_dir`, durably: temp file, fsync, rename +/// over, fsync the directory. Safe both for initial creation and for the +/// in-place promotion stamp. +pub fn write_info(dump_dir: &Path, info: &DumpInfo) -> io::Result<()> { + let content = toml::to_string(info) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("info.toml: {e}")))?; + + let tmp = dump_dir.join(format!("{INFO_FILE}.tmp")); + { + use std::io::Write; + let mut file = std::fs::File::create(&tmp)?; + file.write_all(content.as_bytes())?; + file.sync_all()?; + } + std::fs::rename(&tmp, dump_dir.join(INFO_FILE))?; + std::fs::File::open(dump_dir)?.sync_all()?; + Ok(()) +} + +/// Read and strictly parse `info.toml` from `dump_dir`. Unknown keys +/// (`deny_unknown_fields`), duplicate keys (TOML forbids them), malformed +/// values, and missing required keys all fail loud — a dump with corrupt +/// metadata is not silently usable. The `format_version` gate runs after the +/// parse; there is a single frozen schema (`FORMAT_VERSION`), so a +/// forward-version dump is rejected outright rather than partially read. +pub fn read_info(dump_dir: &Path) -> io::Result { + let content = std::fs::read_to_string(dump_dir.join(INFO_FILE))?; + let bad = |reason: String| io::Error::new(io::ErrorKind::InvalidData, reason); + + let info: DumpInfo = toml::from_str(&content).map_err(|e| bad(format!("info.toml: {e}")))?; + if info.format_version != FORMAT_VERSION { + return Err(bad(format!( + "info.toml: unsupported format_version {} (expected {FORMAT_VERSION})", + info.format_version + ))); + } + Ok(info) +} + +/// Stamp `B` (the promotion's inclusion block) into an existing +/// `info.toml`. Idempotent: re-stamping the same block is a no-op write. +pub fn stamp_promoted_inclusion_block(dump_dir: &Path, block: u64) -> io::Result<()> { + let mut info = read_info(dump_dir)?; + if info.promoted_inclusion_block == Some(block) { + return Ok(()); + } + info.promoted_inclusion_block = Some(block); + write_info(dump_dir, &info) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> DumpInfo { + DumpInfo { + format_version: FORMAT_VERSION, + next_batch_nonce: 7, + l2_tx_index: 123, + promoted_inclusion_block: None, + } + } + + #[test] + fn write_then_read_round_trips_without_promotion() { + let dir = tempfile::tempdir().unwrap(); + write_info(dir.path(), &sample()).unwrap(); + assert_eq!(read_info(dir.path()).unwrap(), sample()); + } + + #[test] + fn stamp_fills_b_and_is_idempotent() { + let dir = tempfile::tempdir().unwrap(); + write_info(dir.path(), &sample()).unwrap(); + + stamp_promoted_inclusion_block(dir.path(), 456).unwrap(); + let stamped = read_info(dir.path()).unwrap(); + assert_eq!(stamped.promoted_inclusion_block, Some(456)); + assert_eq!(stamped.next_batch_nonce, 7, "other fields untouched"); + + stamp_promoted_inclusion_block(dir.path(), 456).unwrap(); + assert_eq!( + read_info(dir.path()).unwrap().promoted_inclusion_block, + Some(456) + ); + } + + #[test] + fn read_rejects_unknown_duplicate_and_missing_keys() { + let dir = tempfile::tempdir().unwrap(); + + std::fs::write(dir.path().join(INFO_FILE), "mystery = 1\n").unwrap(); + assert!(read_info(dir.path()).is_err(), "unknown key must reject"); + + std::fs::write( + dir.path().join(INFO_FILE), + "format_version = 1\nformat_version = 1\n", + ) + .unwrap(); + assert!(read_info(dir.path()).is_err(), "duplicate key must reject"); + + std::fs::write(dir.path().join(INFO_FILE), "format_version = 1\n").unwrap(); + assert!(read_info(dir.path()).is_err(), "missing keys must reject"); + } + + #[test] + fn on_disk_bytes_stay_simple_key_value_lines() { + // The serialized form must remain plain `key = value` integer lines: + // no `[table]` headers, no quoting. This is the operator-readable + // contract and what any previously-written file already looks like. + let dir = tempfile::tempdir().unwrap(); + let mut info = sample(); + info.promoted_inclusion_block = Some(456); + write_info(dir.path(), &info).unwrap(); + let bytes = std::fs::read_to_string(dir.path().join(INFO_FILE)).unwrap(); + assert_eq!( + bytes, + "format_version = 1\nnext_batch_nonce = 7\nl2_tx_index = 123\n\ + promoted_inclusion_block = 456\n" + ); + } + + #[test] + fn read_tolerates_comments_blanks_and_reordering() { + // The robustness win from real TOML over the old line parser: an + // operator inspecting/annotating the file under recovery pressure can + // add comments, blank lines, and reorder keys without bricking it. + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join(INFO_FILE), + "# checkpoint metadata\n\nl2_tx_index = 123\nnext_batch_nonce = 7\n\ + format_version = 1\n", + ) + .unwrap(); + assert_eq!(read_info(dir.path()).unwrap(), sample()); + } + + #[test] + fn read_rejects_wrong_format_version() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join(INFO_FILE), + "format_version = 999\nnext_batch_nonce = 0\nl2_tx_index = 0\n", + ) + .unwrap(); + assert!(read_info(dir.path()).is_err()); + } +} diff --git a/sequencer/src/ingress/inclusion_lane/error.rs b/sequencer/src/ingress/inclusion_lane/error.rs index 0fcdf85..dab7f99 100644 --- a/sequencer/src/ingress/inclusion_lane/error.rs +++ b/sequencer/src/ingress/inclusion_lane/error.rs @@ -7,7 +7,7 @@ use sequencer_core::application::AppError; use thiserror::Error; -use super::snapshot::{GcError, TakeDumpError}; +use super::snapshot::{GcError, StampError, TakeDumpError}; #[derive(Debug, Error)] pub enum InclusionLaneError { @@ -36,6 +36,8 @@ pub enum InclusionLaneError { LoadFromDump(AppError), #[error("snapshot garbage collection failed")] Gc(#[from] GcError), + #[error("stamping promotion metadata into the finalized dump failed")] + PromotionStamp(#[from] StampError), #[error( "no open Tip at lane startup; the runtime must establish it via \ Storage::ensure_open_tip before starting the lane" diff --git a/sequencer/src/ingress/inclusion_lane/mod.rs b/sequencer/src/ingress/inclusion_lane/mod.rs index 7048476..4978eb5 100644 --- a/sequencer/src/ingress/inclusion_lane/mod.rs +++ b/sequencer/src/ingress/inclusion_lane/mod.rs @@ -15,6 +15,7 @@ mod catch_up; mod config; +pub mod dump_info; mod error; mod snapshot; mod types; @@ -34,7 +35,9 @@ use tokio::task::JoinHandle; use crate::runtime::shutdown::ShutdownSignal; use crate::storage::{SafeInputRange, Storage, StoredSafeInput, WriteHead}; -use sequencer_core::application::{AppError, Application, ExecutionOutcome}; +use sequencer_core::application::{ + AppError, Application, ExecutionOutcome, validate_and_execute_user_op, +}; use sequencer_core::l2_tx::DirectInput; use sequencer_core::user_op::SignedUserOp; @@ -83,12 +86,13 @@ impl InclusionLane { let handle = tokio::task::spawn_blocking(move || -> Result<(), InclusionLaneError> { let mut storage = storage; // Single checkpoint selection: the same snapshot supplies both - // the prefix we load the Application from and the offset we + // the dump dir we load the Application from and the offset we // replay from, so the loaded state and the catch-up cursor // can never drift apart. let checkpoint = catch_up_snapshot(&mut storage) .map_err(|source| InclusionLaneError::CatchUp { source })?; - let app = A::from_dump(&checkpoint.prefix).map_err(InclusionLaneError::LoadFromDump)?; + let app = A::from_dump(&dump_info::app_prefix(&checkpoint.dump_dir)) + .map_err(InclusionLaneError::LoadFromDump)?; tracing::debug!( l2_tx_index = checkpoint.l2_tx_index, "inclusion lane resuming from snapshot" @@ -274,6 +278,9 @@ impl InclusionLane { // promotion created garbage — so GC tracks garbage creation, never // starved by load. if promoted { + // Stamp `B` into the freshly finalized dump's info.toml before + // GC (the stamp targets the survivor; GC removes the superseded). + snapshot::stamp_finalized_promotion(&mut self.storage)?; let removed = snapshot::run_gc::(&mut self.storage).map_err(InclusionLaneError::Gc)?; if removed > 0 { @@ -409,6 +416,10 @@ pub(super) enum ChunkOutcome { } fn should_close_batch_by_time(head: &WriteHead, config: &InclusionLaneConfig) -> bool { + // A backwards clock step makes `duration_since` err; `unwrap_or_default` + // then reads as age 0, silently stalling the time-based close trigger + // until the clock catches up (review F8). Acceptable: the size trigger + // is unaffected, and a wedge here is liveness-only, never correctness. let age = SystemTime::now() .duration_since(head.batch_created_at) .unwrap_or_default(); @@ -419,12 +430,15 @@ fn execute_user_op( app: &mut impl Application, item: PendingUserOp, current_frame_fee: u16, + frame_safe_block: u64, included: &mut Vec, ) -> Result<(), InclusionLaneError> { - match app.validate_and_execute_user_op( + match validate_and_execute_user_op( + app, item.signed.sender, &item.signed.user_op, current_frame_fee, + frame_safe_block, ) { Ok(ExecutionOutcome::Included { .. }) => included.push(item), Ok(ExecutionOutcome::Invalid(reason)) => { @@ -432,6 +446,14 @@ fn execute_user_op( .respond_to .send(Err(SequencerError::invalid(reason.to_string()))); } + // Fail loud — the lane half of an asymmetry with the canonical fold + // (`execute_frame_user_ops` in `sequencer-core`), which silently *skips* + // this same `AppError`. Duality (I1) is preserved: an `AppError` excludes + // the op from state on both sides (here it is never pushed to `included`, + // there `outputs` is never extended), so the canonical state agrees. The + // lane additionally aborts because an error from a *validated* op is an + // internal-invariant breach, not a user-facing rejection — dead by + // construction today (see the matching note in the scheduler). Err(err) => { let reason = match &err { AppError::Internal { reason } => reason.clone(), @@ -464,7 +486,7 @@ pub(super) fn dequeue_and_execute_user_op_chunk( while executed < max_chunk { match rx.try_recv() { Ok(item) => { - execute_user_op(app, item, current_frame_fee, included)?; + execute_user_op(app, item, current_frame_fee, head.safe_block, included)?; executed = executed.saturating_add(1); let projected = head diff --git a/sequencer/src/ingress/inclusion_lane/snapshot.rs b/sequencer/src/ingress/inclusion_lane/snapshot.rs index c2d3251..8c219ce 100644 --- a/sequencer/src/ingress/inclusion_lane/snapshot.rs +++ b/sequencer/src/ingress/inclusion_lane/snapshot.rs @@ -32,8 +32,9 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; -use sequencer_core::application::{AppError, Application}; +use sequencer_core::application::Application; +use super::dump_info::{self, DumpInfo}; use crate::storage::{SafeInputRange, Storage, WriteHead}; /// Errors from snapshot-taking at batch close. @@ -41,8 +42,8 @@ use crate::storage::{SafeInputRange, Storage, WriteHead}; pub enum TakeDumpError { #[error("storage: {0}")] Storage(#[from] rusqlite::Error), - #[error("app: {0}")] - App(#[from] AppError), + #[error(transparent)] + CreateDump(#[from] dump_info::CreateDumpDirError), } /// Errors from the post-promotion GC pass. @@ -52,6 +53,16 @@ pub enum GcError { Storage(#[from] rusqlite::Error), } +/// Errors from stamping promotion metadata (`B`) into the finalized +/// dump's `info.toml`. +#[derive(Debug, thiserror::Error)] +pub enum StampError { + #[error("storage: {0}")] + Storage(#[from] rusqlite::Error), + #[error("io: {0}")] + Io(#[from] std::io::Error), +} + /// Run one garbage-collection pass: delete every unreferenced dump /// row in SQLite (atomically) and best-effort delete the corresponding /// directories on disk. Filesystem failures log and continue — an @@ -61,7 +72,7 @@ pub enum GcError { pub(super) fn run_gc(storage: &mut Storage) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { - if let Err(err) = A::delete_dump(&row.prefix) { + if let Err(err) = dump_info::delete_dump_dir::(&row.prefix) { tracing::warn!( error = %err, prefix = ?row.prefix, @@ -72,14 +83,35 @@ pub(super) fn run_gc(storage: &mut Storage) -> Result< Ok(removed.len()) } +/// Stamp `B` — the promotion's L1 inclusion block — into the freshly +/// finalized dump's `info.toml`, completing the checkpoint metadata +/// whose other fields were written at batch close. During live operation +/// the DB row is authoritative for `B` and `info.toml` mirrors it (a +/// crash between the promoting commit and this stamp is healed by the +/// idempotent startup re-stamp from the same row). But `info.toml` is +/// also the durable checkpoint the operator backs up and `setup +/// --recovery` reads — there the DB is gone and the file is the sole +/// authority for the resume nonce `N` — so keeping the two coherent here +/// is load-bearing, not cosmetic. +pub(super) fn stamp_finalized_promotion(storage: &mut Storage) -> Result<(), StampError> { + let finalized = storage + .finalized_dump()? + .expect("a promotion just committed, so the finalized snapshot row exists"); + dump_info::stamp_promoted_inclusion_block(&finalized.dump.prefix, finalized.inclusion_block)?; + Ok(()) +} + /// Close the current batch and register its snapshot atomically. /// /// Order matters for crash/error safety: -/// 1. Read the closing batch's nonce (assigned at open) and build a -/// unique dump prefix. -/// 2. [`Application::create_dump`] writes + fsyncs the dump **before** -/// any DB mutation. On failure nothing is sealed — the batch stays -/// the open Tip and the lane retries on its next pass. +/// 1. Read the closing batch's nonce (assigned at open) and the replay +/// head; build a unique dump directory. +/// 2. [`dump_info::create_dump_dir_with_info`] writes + fsyncs the dir, +/// `info.toml` (`N` = nonce + 1, the replay head; `B` stamped later +/// at promotion), and the app's dump — all **before** any DB +/// mutation. On failure nothing is sealed — the batch stays the open +/// Tip; the error propagates per the lane's fail-loud policy (the +/// retry happens on the next boot, after catch-up). /// 3. One DB transaction seals the batch, opens the next, and inserts /// the `pending_snapshots` row /// ([`Storage::close_frame_and_batch_with_pending_dump`]). A @@ -98,9 +130,23 @@ pub(super) fn close_batch_with_snapshot( dumps_dir: &Path, ) -> Result<(), TakeDumpError> { let nonce = storage.batch_nonce(head.batch_index)?; - let prefix = make_dump_prefix(dumps_dir, nonce); - app.create_dump(&prefix)?; - storage.close_frame_and_batch_with_pending_dump(head, next_safe_block, &prefix, nonce)?; + // The snapshot reflects state through the global valid replay head + // as of the close; single-writer lane, so the head can't move before + // the close transaction (which re-asserts equality). + let l2_tx_index = storage.valid_ordered_l2_tx_head()?; + let dump_dir = make_dump_dir(dumps_dir, nonce); + dump_info::create_dump_dir_with_info( + app, + &dump_dir, + &DumpInfo::at_batch_close(nonce, l2_tx_index), + )?; + storage.close_frame_and_batch_with_pending_dump( + head, + next_safe_block, + &dump_dir, + nonce, + l2_tx_index, + )?; Ok(()) } @@ -121,9 +167,13 @@ pub(super) fn take_dump_at_batch_close( // not just this batch's own rows — so an empty batch correctly // records the prior head rather than genesis. let l2_tx_index = storage.valid_ordered_l2_tx_head()?; - let prefix = make_dump_prefix(dumps_dir, nonce); - app.create_dump(&prefix)?; - storage.insert_pending_dump(&prefix, nonce, l2_tx_index)?; + let dump_dir = make_dump_dir(dumps_dir, nonce); + dump_info::create_dump_dir_with_info( + app, + &dump_dir, + &DumpInfo::at_batch_close(nonce, l2_tx_index), + )?; + storage.insert_pending_dump(&dump_dir, nonce, l2_tx_index)?; Ok(()) } @@ -206,10 +256,12 @@ impl BlockObservation { } } -fn make_dump_prefix(dumps_dir: &Path, nonce: u64) -> PathBuf { +fn make_dump_dir(dumps_dir: &Path, nonce: u64) -> PathBuf { // Unique per call within a process: nonce + nanos + atomic counter. // Nonces can be reused across recovery cascades, so they alone - // don't guarantee uniqueness; the nanos+counter pair does. + // don't guarantee uniqueness; the nanos+counter pair does. The name + // is opaque — checkpoint metadata lives in the dir's `info.toml`, + // never in the path. static COUNTER: AtomicU64 = AtomicU64::new(0); let counter = COUNTER.fetch_add(1, Ordering::Relaxed); let nanos = SystemTime::now() @@ -224,7 +276,7 @@ mod tests { use std::path::{Path, PathBuf}; use std::sync::Mutex; - use alloy_primitives::{Address, U256}; + use alloy_primitives::Address; use sequencer_core::application::{AppError, AppOutputs, Application, InvalidReason}; use sequencer_core::l2_tx::ValidUserOp; use sequencer_core::user_op::UserOp; @@ -232,6 +284,7 @@ mod tests { use crate::storage::Storage; use crate::storage::test_helpers::{seed_closed_batches, temp_db}; + use super::dump_info; use super::{BlockObservation, take_dump_at_batch_close}; /// Minimal Application that records every `create_dump` call. The @@ -255,14 +308,6 @@ mod tests { impl Application for RecordingDumpApp { const MAX_METHOD_PAYLOAD_BYTES: usize = 0; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::ZERO - } - fn validate_user_op( &self, _sender: Address, @@ -275,10 +320,26 @@ mod tests { fn execute_valid_user_op( &mut self, _user_op: &ValidUserOp, + _safe_block: u64, ) -> Result { Ok(Vec::new()) } + fn execute_direct_input( + &mut self, + _input: &sequencer_core::l2_tx::DirectInput, + ) -> Result { + unimplemented!("not used in these tests") + } + + fn executed_input_count(&self) -> u64 { + 0 + } + + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(_prefix: &Path) -> Result { unimplemented!("not used in these tests") } @@ -327,14 +388,25 @@ mod tests { let recorded = app.recorded(); assert_eq!(recorded.len(), 1); - let prefix = &recorded[0]; - assert!(prefix.starts_with(dumps_dir.path())); - assert!(prefix.join("state").exists(), "dump's state file exists"); + let app_prefix = &recorded[0]; + assert!(app_prefix.starts_with(dumps_dir.path())); + assert!( + app_prefix.join("state").exists(), + "dump's state file exists" + ); let pending = storage.latest_pending_dump().unwrap().unwrap(); assert_eq!(pending.nonce, 1); - assert_eq!(pending.dump.prefix, *prefix); + // The DB row stores the dump dir; the app dumped into `state`. + assert_eq!(dump_info::app_prefix(&pending.dump.prefix), *app_prefix); assert_eq!(pending.l2_tx_index, 0); // empty batch → no L2 txs + + // info.toml carries the resume nonce and the same cursor; + // B is unstamped until promotion. + let info = dump_info::read_info(&pending.dump.prefix).unwrap(); + assert_eq!(info.next_batch_nonce, 2); + assert_eq!(info.l2_tx_index, 0); + assert_eq!(info.promoted_inclusion_block, None); } #[test] @@ -417,14 +489,6 @@ mod tests { impl Application for FailingDumpApp { const MAX_METHOD_PAYLOAD_BYTES: usize = 0; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::ZERO - } - fn validate_user_op( &self, _sender: Address, @@ -437,10 +501,26 @@ mod tests { fn execute_valid_user_op( &mut self, _user_op: &ValidUserOp, + _safe_block: u64, ) -> Result { Ok(Vec::new()) } + fn execute_direct_input( + &mut self, + _input: &sequencer_core::l2_tx::DirectInput, + ) -> Result { + unimplemented!("not used in these tests") + } + + fn executed_input_count(&self) -> u64 { + 0 + } + + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(_prefix: &Path) -> Result { Ok(FailingDumpApp) } @@ -474,7 +554,10 @@ mod tests { let err = super::close_batch_with_snapshot(&app, &mut storage, &mut head, 0, dumps_dir.path()) .expect_err("create_dump failure must abort the close"); - assert!(matches!(err, super::TakeDumpError::App(_))); + assert!(matches!( + err, + super::TakeDumpError::CreateDump(dump_info::CreateDumpDirError::App(_)) + )); // Nothing sealed: the batch is still the open Tip, no successor, // and no pending snapshot row was written. diff --git a/sequencer/src/ingress/inclusion_lane/tests.rs b/sequencer/src/ingress/inclusion_lane/tests.rs index 808a5be..c21a167 100644 --- a/sequencer/src/ingress/inclusion_lane/tests.rs +++ b/sequencer/src/ingress/inclusion_lane/tests.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; -use alloy_primitives::{Address, Signature, U256}; +use alloy_primitives::{Address, Signature}; use app_core::application::MAX_METHOD_PAYLOAD_BYTES as WALLET_MAX_METHOD_PAYLOAD_BYTES; use rusqlite::params; use tokio::sync::{mpsc, oneshot}; @@ -31,14 +31,6 @@ struct TestApp { impl Application for TestApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, sender: Address) -> u32 { - self.nonces.get(&sender).copied().unwrap_or(0) - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::MAX - } - fn validate_user_op( &self, _sender: Address, @@ -48,8 +40,13 @@ impl Application for TestApp { Ok(()) } - fn execute_valid_user_op(&mut self, user_op: &ValidUserOp) -> Result { - let next_nonce = self.current_user_nonce(user_op.sender).wrapping_add(1); + fn execute_valid_user_op( + &mut self, + user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { + let current = self.nonces.get(&user_op.sender).copied().unwrap_or(0); + let next_nonce = current.wrapping_add(1); self.nonces.insert(user_op.sender, next_nonce); self.executed_input_count = self.executed_input_count.saturating_add(1); Ok(Vec::new()) @@ -64,6 +61,10 @@ impl Application for TestApp { self.executed_input_count } + fn last_executed_safe_block(&self) -> u64 { + 0 + } + // The lane loads its app via `from_dump` after the runtime // registers a genesis dump, so test stubs must reload to the // initial state (an empty `nonces` map). `create_dump` writes @@ -94,14 +95,6 @@ struct InternalUserOpApp; impl Application for InternalUserOpApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::MAX - } - fn validate_user_op( &self, _sender: Address, @@ -111,12 +104,28 @@ impl Application for InternalUserOpApp { Ok(()) } - fn execute_valid_user_op(&mut self, _user_op: &ValidUserOp) -> Result { + fn execute_valid_user_op( + &mut self, + _user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { Err(AppError::Internal { reason: "app invariant failed".to_string(), }) } + fn execute_direct_input(&mut self, _input: &DirectInput) -> Result { + unimplemented!("not used in these tests") + } + + fn executed_input_count(&self) -> u64 { + 0 + } + + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(_prefix: &Path) -> Result { Ok(InternalUserOpApp) } @@ -176,14 +185,6 @@ impl SharedCountingApp { impl Application for SharedCountingApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::MAX - } - fn validate_user_op( &self, _sender: Address, @@ -193,7 +194,11 @@ impl Application for SharedCountingApp { Ok(()) } - fn execute_valid_user_op(&mut self, _user_op: &ValidUserOp) -> Result { + fn execute_valid_user_op( + &mut self, + _user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { Ok(Vec::new()) } @@ -202,6 +207,14 @@ impl Application for SharedCountingApp { Ok(Vec::new()) } + fn executed_input_count(&self) -> u64 { + self.executed_direct_inputs + } + + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(prefix: &Path) -> Result { let bytes = std::fs::read(Self::state_file_in_dump(prefix))?; let counter = @@ -255,14 +268,6 @@ impl Default for ReplayRecordingApp { impl Application for ReplayRecordingApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::MAX - } - fn validate_user_op( &self, _sender: Address, @@ -272,7 +277,11 @@ impl Application for ReplayRecordingApp { Ok(()) } - fn execute_valid_user_op(&mut self, user_op: &ValidUserOp) -> Result { + fn execute_valid_user_op( + &mut self, + user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { self.replayed.push(ReplayEvent::UserOp { sender: user_op.sender, data: user_op.data.clone(), @@ -295,6 +304,10 @@ impl Application for ReplayRecordingApp { self.executed_input_count } + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(_prefix: &Path) -> Result { Ok(Self::default()) } @@ -344,10 +357,20 @@ fn register_genesis_snapshot(app: &A, storage: &mut Storage, dum // tests that re-seed storage), so reuse-free naming matters. static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let counter = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let prefix = dumps_dir.join(format!("genesis-{counter}")); - app.create_dump(&prefix).expect("create genesis dump"); + let dump_dir = dumps_dir.join(format!("genesis-{counter}")); + super::dump_info::create_dump_dir_with_info( + app, + &dump_dir, + &super::dump_info::DumpInfo { + format_version: super::dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + l2_tx_index: 0, + promoted_inclusion_block: Some(0), + }, + ) + .expect("create genesis dump"); storage - .insert_finalized_dump(&prefix, 0, 0) + .insert_finalized_dump(&dump_dir, 0, 0) .expect("insert finalized snapshot"); } @@ -735,7 +758,7 @@ async fn safe_inputs_already_available_are_sequenced_before_later_user_ops() { .ordered_l2_txs_page_from(0, 1_000_000) .expect("load ordered replay") .into_iter() - .map(|(_offset, tx)| tx) + .map(|(_offset, tx, _frame_safe_block)| tx) .collect() }; shutdown_lane(&shutdown, lane_handle).await; @@ -972,14 +995,6 @@ impl UserOpCounterApp { impl Application for UserOpCounterApp { const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; - fn current_user_nonce(&self, _sender: Address) -> u32 { - 0 - } - - fn current_user_balance(&self, _sender: Address) -> U256 { - U256::MAX - } - fn validate_user_op( &self, _sender: Address, @@ -989,15 +1004,27 @@ impl Application for UserOpCounterApp { Ok(()) } - fn execute_valid_user_op(&mut self, _user_op: &ValidUserOp) -> Result { + fn execute_valid_user_op( + &mut self, + _user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { self.executed_user_ops = self.executed_user_ops.saturating_add(1); Ok(Vec::new()) } + fn execute_direct_input(&mut self, _input: &DirectInput) -> Result { + unimplemented!("not used in these tests") + } + fn executed_input_count(&self) -> u64 { self.executed_user_ops } + fn last_executed_safe_block(&self) -> u64 { + 0 + } + fn from_dump(prefix: &Path) -> Result { let bytes = std::fs::read(Self::state_file_in_dump(prefix))?; let count = @@ -1033,8 +1060,9 @@ impl Application for UserOpCounterApp { } } -fn read_dump_counter(prefix: &Path) -> u64 { - let bytes = std::fs::read(prefix.join("state")).expect("read dump state file"); +fn read_dump_counter(dump_dir: &Path) -> u64 { + let state_file = UserOpCounterApp::state_file_in_dump(&super::dump_info::app_prefix(dump_dir)); + let bytes = std::fs::read(state_file).expect("read dump state file"); u64::from_le_bytes( bytes .as_slice() diff --git a/sequencer/src/l1/mod.rs b/sequencer/src/l1/mod.rs index 9300410..fa72110 100644 --- a/sequencer/src/l1/mod.rs +++ b/sequencer/src/l1/mod.rs @@ -9,3 +9,4 @@ pub mod partition; pub mod provider; pub mod reader; pub mod submitter; +pub mod watermark; diff --git a/sequencer/src/l1/partition.rs b/sequencer/src/l1/partition.rs index 09383a1..1f80708 100644 --- a/sequencer/src/l1/partition.rs +++ b/sequencer/src/l1/partition.rs @@ -21,8 +21,14 @@ use async_recursion::async_recursion; use cartesi_rollups_contracts::input_box::InputBox::InputAdded; use cartesi_rollups_contracts::inputs::Inputs::EvmAdvanceCall; +/// Fetch every `InputAdded` log for `app_address_filter`, splitting the range +/// on RPC long-range errors. The result is in **RPC-return order** — partition +/// sub-ranges are concatenated and a single `eth_getLogs` may itself reorder — +/// so any consumer that relies on L1 order must sort. Prefer +/// [`get_input_added_events_ordered`], which folds that sort in; this raw form +/// is the partition primitive (and the recursion's own building block). #[async_recursion] -pub async fn get_input_added_events( +pub(crate) async fn get_input_added_events( provider: &impl Provider, app_address_filter: Address, input_box_address: &Address, @@ -175,6 +181,38 @@ pub fn sort_logs_by_l1_order( }); } +/// [`get_input_added_events`] with the logs returned in canonical L1 order +/// `(block, tx_index, log_index)`. The single fetch path for every consumer that +/// depends on L1 ordering — the reader's dense-index contiguity witness and the +/// submitter's batch-nonce fold both require it — so the sort lives here once +/// instead of being duplicated (and forgettable) at each call site. Matches the +/// order the watchdog applies on its side. +pub(crate) async fn get_input_added_events_ordered( + provider: &impl Provider, + app_address_filter: Address, + input_box_address: &Address, + start_block: u64, + end_block: u64, + long_block_range_error_codes: &[String], +) -> Result, Vec> { + let mut events = get_input_added_events( + provider, + app_address_filter, + input_box_address, + start_block, + end_block, + long_block_range_error_codes, + ) + .await?; + sort_logs_by_l1_order( + &mut events, + |(_, log)| log.block_number.unwrap_or(0), + |(_, log)| log.transaction_index.unwrap_or(0), + |(_, log)| log.log_index.unwrap_or(0), + ); + Ok(events) +} + pub fn decode_evm_advance_input(input: &[u8]) -> Result { EvmAdvanceCall::abi_decode(input).map_err(|err| err.to_string()) } diff --git a/sequencer/src/l1/provider.rs b/sequencer/src/l1/provider.rs index 672c8b9..7c06f60 100644 --- a/sequencer/src/l1/provider.rs +++ b/sequencer/src/l1/provider.rs @@ -78,6 +78,51 @@ pub fn create_signer_provider(url: &str, private_key: &str) -> Result Result { + let provider = + create_signer_provider(url, private_key).map_err(VerifiedSignerProviderError::Create)?; + let rpc_chain_id = provider + .get_chain_id() + .await + .map_err(|e| VerifiedSignerProviderError::ChainIdRpc(e.to_string()))?; + if rpc_chain_id != expected_chain_id { + return Err(VerifiedSignerProviderError::ChainIdMismatch { + rpc: rpc_chain_id, + expected: expected_chain_id, + }); + } + Ok(provider) +} + #[cfg(test)] mod tests { use super::*; @@ -153,4 +198,51 @@ mod tests { create_signer_provider("http://127.0.0.1:8545", good_key) .expect("valid key must be accepted"); } + + // ── Keyed-write chain-id gate (review): the guarded constructor must + // confirm the served chain matches the pinned one before signing ─ + + fn require_anvil() { + assert!( + std::process::Command::new("anvil") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok(), + "anvil not found on PATH — install Foundry (https://getfoundry.sh)" + ); + } + + /// Anvil account 0 — a valid signing key for the guarded constructor. + const ANVIL_KEY_0: &str = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + + #[tokio::test] + async fn verified_signer_provider_gates_on_chain_id() { + use alloy::node_bindings::Anvil; + + require_anvil(); + let anvil = Anvil::default().spawn(); + let chain_id = anvil.chain_id(); + + // Matching chain id → a usable signing provider. + create_verified_signer_provider(&anvil.endpoint(), ANVIL_KEY_0, chain_id) + .await + .expect("matching chain id yields a verified signer provider"); + + // Wrong pinned chain id → terminal ChainIdMismatch, before any signing, + // surfacing the served id vs the pinned one. + let err = create_verified_signer_provider(&anvil.endpoint(), ANVIL_KEY_0, chain_id + 1) + .await + .expect_err("a mismatched pinned chain id must be rejected before signing"); + assert!( + matches!( + err, + VerifiedSignerProviderError::ChainIdMismatch { rpc, expected } + if rpc == chain_id && expected == chain_id + 1 + ), + "expected ChainIdMismatch {{ rpc: {chain_id}, expected: {} }}, got {err:?}", + chain_id + 1 + ); + } } diff --git a/sequencer/src/l1/reader.rs b/sequencer/src/l1/reader.rs index 64afb25..7a9384b 100644 --- a/sequencer/src/l1/reader.rs +++ b/sequencer/src/l1/reader.rs @@ -18,9 +18,9 @@ use cartesi_rollups_contracts::input_box::InputBox; use tokio::task::JoinHandle; use tracing::info; -use crate::l1::partition::{decode_evm_advance_input, get_input_added_events}; +use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::runtime::shutdown::ShutdownSignal; -use crate::storage::{Storage, StorageOpenError, StoredSafeInput}; +use crate::storage::{FrontierMode, Storage, StorageOpenError, StoredSafeInput}; use sequencer_core::protocol::ProtocolTiming; #[derive(Debug, Clone)] @@ -30,12 +30,22 @@ pub struct InputReaderConfig { pub poll_interval: Duration, /// Error codes that trigger `get_logs` retries with a shorter block range. pub long_block_range_error_codes: Vec, + /// The chain id `setup` pinned (`run`) / is pinning (`setup`). The reader + /// verifies the provider actually serves this chain on its first successful + /// contact — the RPC URL is an operator CLI/env arg that may be repointed + /// across restarts (token rotation, provider swap), so `run` cannot assume + /// it still matches the pinned identity. This backstops the boot-time check + /// ([`crate::runtime::validate_rpc_chain_id`]), which is skipped when L1 is + /// unreachable at boot. + pub expected_chain_id: u64, } #[derive(Debug, thiserror::Error)] pub enum InputReaderError { #[error("provider/transport: {0}")] Provider(String), + #[error("RPC chain id {rpc} does not match the pinned chain id {expected}")] + ChainIdMismatch { rpc: u64, expected: u64 }, #[error("bootstrap: {0}")] Bootstrap(String), #[error(transparent)] @@ -58,6 +68,15 @@ pub struct InputReader { /// Protocol timing used to keep `safe_accepted_batches` consistent with /// every `append_safe_inputs` write. timing: ProtocolTiming, + /// Whether each sync also populates the scheduler-accepted frontier + /// ([`FrontierMode::Populate`] normally; `setup --recovery` defers it — see + /// [`Self::set_frontier_mode`]). + frontier_mode: FrontierMode, + /// Set once the provider's chain id has been verified against + /// [`InputReaderConfig::expected_chain_id`] on a successful RPC contact. + /// Until then every `advance_once` re-attempts the check, so an unreachable + /// L1 keeps retrying and the verification fires the moment L1 returns. + chain_id_verified: bool, } impl InputReader { @@ -115,9 +134,19 @@ impl InputReader { db_path, batch_submitter, timing, + frontier_mode: FrontierMode::Populate, + chain_id_verified: false, } } + /// Set whether subsequent syncs populate the scheduler-accepted frontier. + /// `setup --recovery` sets [`FrontierMode::DeferUntilAnchorSet`] for the + /// syncs that feed the fold (the frontier is populated by `run`'s first + /// sync, once the anchor = `N'`). + pub fn set_frontier_mode(&mut self, mode: FrontierMode) { + self.frontier_mode = mode; + } + pub fn input_box_address(&self) -> Address { self.input_box_address } @@ -182,6 +211,11 @@ impl InputReader { &mut self, provider: &impl Provider, ) -> Result<(), InputReaderError> { + // Verify the provider serves the pinned chain before reading anything + // from it — a wrong-chain RPC's address-filtered logs would otherwise + // flow into `safe_inputs`. + self.verify_chain_id(provider).await?; + let current_safe_head = latest_safe_head(provider).await?; let current_safe_block = current_safe_head.block_number; let previous_safe_block = self.current_safe_block().await?; @@ -201,7 +235,12 @@ impl InputReader { } let start_block = scan_floor + 1; - let events = get_input_added_events( + // L1-01: the dense-index contiguity witness below requires L1 event + // order. `get_input_added_events_ordered` guarantees it — a raw + // `eth_getLogs` reorder would otherwise turn the check into a permanent + // retry loop. The InputBox `index` is monotonic with L1 order, so the + // sorted stream aligns `onchain_indices` with `expected_start + i`. + let events = get_input_added_events_ordered( provider, self.config.app_address, &self.input_box_address, @@ -220,8 +259,22 @@ impl InputReader { )) })?; + // F5: the InputBox assigns every input a per-app, gap-free `index`. We + // ingest every event for this app from genesis, so each input's on-chain + // index must equal the dense local `safe_input_index` it is assigned + // (= the running count of already-stored inputs). Collect the on-chain + // indices and verify contiguity below, *before* persisting — a gap means + // the provider returned an incomplete `get_logs` set (a clamped or + // lagging-replica response), and advancing the safe head past a missing + // input would let the scheduler force-drain it while we never saw it. + let expected_start = self.safe_input_end_exclusive().await?; let mut batch = Vec::with_capacity(events.len()); + let mut onchain_indices = Vec::with_capacity(events.len()); for (event, log) in events { + onchain_indices.push(u64::try_from(event.index).map_err(|_| { + InputReaderError::Provider("InputAdded index exceeds u64".to_string()) + })?); + let block_number = log.block_number.ok_or_else(|| { InputReaderError::Provider("InputAdded log missing block_number".to_string()) })?; @@ -241,6 +294,29 @@ impl InputReader { }); } + // Fail loud on a hole rather than persist a partial set (retryable: a + // consistent provider returns the full set on the next tick). + check_input_index_contiguity(expected_start, &onchain_indices)?; + + // Completeness witness. Contiguity above proves the returned rows are the + // right *prefix*; this proves the prefix is *complete*. A truncated + // `get_logs` (still a contiguous prefix) shows up as an InputBox input + // count — pinned at the scanned safe block — that exceeds what we + // received. Without it, a dropped tail deposit `≤` the safe head would be + // persisted as complete and only caught on the *next* input, after the + // lane may have stamped frames past it (divergence). Pinning the count to + // `current_safe_block` (not latest) keeps it consistent with the scanned + // range and forces the serving node to actually have that block's state. + let onchain_count = input_count_at_block( + provider, + &self.input_box_address, + self.config.app_address, + current_safe_block, + ) + .await?; + let received_total = expected_start.saturating_add(batch.len() as u64); + check_input_count_complete(current_safe_block, received_total, onchain_count)?; + info!( block_range = %format!("{}..={}", start_block, current_safe_block), count = batch.len(), @@ -250,6 +326,32 @@ impl InputReader { self.append_safe_inputs(current_safe_head, batch).await } + /// Verify the provider serves the pinned chain id, once per reader instance, + /// on the first successful contact. A transport failure is retryable + /// (`Provider`) and leaves the flag unset, so the check re-fires on the next + /// tick until L1 is reachable; a value mismatch is fatal (`ChainIdMismatch`) + /// and propagates out of the run loop / aborts a recovery sync. This is the + /// backstop for `run`'s boot-time check, which is skipped when L1 is + /// unreachable at boot — without it, an RPC reconnecting on the wrong chain + /// would ingest address-filtered foreign logs unnoticed. + async fn verify_chain_id(&mut self, provider: &impl Provider) -> Result<(), InputReaderError> { + if self.chain_id_verified { + return Ok(()); + } + let rpc_chain_id = provider + .get_chain_id() + .await + .map_err(|e| InputReaderError::Provider(e.to_string()))?; + if rpc_chain_id != self.config.expected_chain_id { + return Err(InputReaderError::ChainIdMismatch { + rpc: rpc_chain_id, + expected: self.config.expected_chain_id, + }); + } + self.chain_id_verified = true; + Ok(()) + } + async fn current_safe_block(&self) -> Result, InputReaderError> { let db_path = self.db_path.clone(); tokio::task::spawn_blocking(move || { @@ -260,6 +362,20 @@ impl InputReader { .map_err(|err| InputReaderError::Join(err.to_string()))? } + /// Count of already-stored safe inputs (= the next local `safe_input_index`). + /// Used as the expected on-chain index of the next ingested input (F5). + async fn safe_input_end_exclusive(&self) -> Result { + let db_path = self.db_path.clone(); + tokio::task::spawn_blocking(move || { + let mut storage = Storage::open(&db_path)?; + storage + .safe_input_end_exclusive() + .map_err(InputReaderError::from) + }) + .await + .map_err(|err| InputReaderError::Join(err.to_string()))? + } + async fn append_safe_inputs( &self, current_safe_head: SafeHead, @@ -268,6 +384,7 @@ impl InputReader { let db_path = self.db_path.clone(); let batch_submitter = self.batch_submitter; let timing = self.timing; + let frontier_mode = self.frontier_mode; tokio::task::spawn_blocking(move || { let mut storage = Storage::open(&db_path)?; storage @@ -277,6 +394,7 @@ impl InputReader { &batch, batch_submitter, &timing, + frontier_mode, ) .map_err(InputReaderError::from) }) @@ -317,6 +435,76 @@ struct SafeHead { block_timestamp: u64, } +/// Verify the ingested InputBox indices continue contiguously from +/// `expected_start` (the running count of stored inputs). The InputBox assigns +/// every input a per-app, gap-free index, and we ingest every event for this app +/// from genesis, so the indices must be `expected_start, expected_start+1, …`. A +/// gap means the provider returned an incomplete `get_logs` set (a clamped or +/// lagging-replica response, review F5); returning a retryable [`Provider`] error +/// refuses to persist the hole — a consistent provider returns the full set on +/// the next tick. +/// +/// [`Provider`]: InputReaderError::Provider +fn check_input_index_contiguity( + expected_start: u64, + onchain_indices: &[u64], +) -> Result<(), InputReaderError> { + for (offset, &index) in onchain_indices.iter().enumerate() { + let expected = expected_start.saturating_add(offset as u64); + if index != expected { + return Err(InputReaderError::Provider(format!( + "non-contiguous InputBox index: expected {expected}, got {index} — \ + provider returned an incomplete InputAdded set (review F5)" + ))); + } + } + Ok(()) +} + +/// Verify the InputBox's own input count at the scanned safe block matches the +/// number of inputs we now hold (`received_total` = previously stored + just +/// received). A mismatch means the `get_logs` response was an incomplete prefix +/// — typically a truncated tail a contiguity check alone cannot see (review F5). +/// Retryable [`Provider`] error: a consistent provider returns the full set, and +/// the matching count, on the next tick. +/// +/// [`Provider`]: InputReaderError::Provider +fn check_input_count_complete( + safe_block: u64, + received_total: u64, + onchain_count: u64, +) -> Result<(), InputReaderError> { + if onchain_count != received_total { + return Err(InputReaderError::Provider(format!( + "InputBox input count at block {safe_block} is {onchain_count}, expected \ + {received_total} — provider returned an incomplete get_logs set (review F5)" + ))); + } + Ok(()) +} + +/// The InputBox's per-app input count, pinned at `block`. Used as the F5 +/// completeness witness — see [`check_input_count_complete`]. Pinning to the +/// scanned safe block (not `latest`) keeps it consistent with the `get_logs` +/// range and forces the serving node to have that block's state. +async fn input_count_at_block( + provider: &impl Provider, + input_box_address: &Address, + app_address: Address, + block: u64, +) -> Result { + let input_box = InputBox::new(*input_box_address, provider); + let count = input_box + .getNumberOfInputs(app_address) + .block(alloy::eips::BlockId::number(block)) + .call() + .await + .map_err(|e| InputReaderError::Provider(e.to_string()))?; + u64::try_from(count).map_err(|_| { + InputReaderError::Provider("InputBox getNumberOfInputs exceeds u64".to_string()) + }) +} + async fn latest_safe_head(provider: &impl Provider) -> Result { let block = provider .get_block(Safe.into()) @@ -357,6 +545,9 @@ mod tests { app_address: Address::ZERO, poll_interval, long_block_range_error_codes: Vec::new(), + // Anvil's default chain id — tests that drive a real provider go + // through the chain-id check in `advance_once`. + expected_chain_id: 31337, }, Address::ZERO, genesis_block, @@ -459,6 +650,53 @@ mod tests { ); } + #[tokio::test] + async fn advance_once_refuses_wrong_chain_id() { + require_anvil(); + + let anvil = Anvil::default().block_time(1).timeout(30_000).spawn(); + let db_file = NamedTempFile::new().expect("temp file"); + // Reader pinned to a chain id the Anvil provider does not serve. + let mut reader = InputReader::from_parts( + InputReaderConfig { + rpc_url: anvil.endpoint_url().to_string(), + app_address: Address::ZERO, + poll_interval: Duration::from_secs(1), + long_block_range_error_codes: Vec::new(), + expected_chain_id: 999_999, + }, + Address::ZERO, + 0, + db_file.path().to_string_lossy().into_owned(), + Address::ZERO, + test_timing(), + ); + let provider = alloy::providers::ProviderBuilder::new() + .connect(anvil.endpoint_url().to_string().as_str()) + .await + .expect("connect provider"); + + let result = reader.advance_once(&provider).await; + assert!( + matches!( + result, + Err(InputReaderError::ChainIdMismatch { rpc, expected }) + if rpc == 31337 && expected == 999_999 + ), + "expected ChainIdMismatch, got {result:?}" + ); + + // The check runs before any read/persist, so a wrong-chain provider + // must not have written a safe-head row. + let mut storage = + Storage::open(db_file.path().to_string_lossy().as_ref()).expect("open storage"); + assert_eq!( + storage.current_safe_block().expect("read safe block"), + None, + "a chain-id mismatch must abort before persisting any L1 view" + ); + } + #[tokio::test] async fn current_safe_block_is_unknown_before_first_observation() { let db_file = NamedTempFile::new().expect("temp file"); @@ -509,6 +747,7 @@ mod tests { app_address: Address::ZERO, poll_interval: Duration::from_secs(1), long_block_range_error_codes: Vec::new(), + expected_chain_id: 31337, }, Address::ZERO, test_timing(), @@ -609,4 +848,65 @@ mod tests { .contains("unsupported DataAvailability.InputBoxAndEspresso") ); } + + #[test] + fn input_index_contiguity_accepts_a_gap_free_run() { + assert!(check_input_index_contiguity(0, &[0, 1, 2]).is_ok()); + // Continues contiguously from a non-zero count of already-stored inputs. + assert!(check_input_index_contiguity(7, &[7, 8, 9]).is_ok()); + // An empty batch (no new inputs) is trivially contiguous. + assert!(check_input_index_contiguity(5, &[]).is_ok()); + } + + #[test] + fn input_index_contiguity_rejects_a_dropped_middle_input() { + // Provider returned 7,9 — input 8 was dropped (clamped/lagging get_logs). + let err = check_input_index_contiguity(7, &[7, 9]) + .expect_err("a dropped middle input must be rejected"); + assert!( + matches!(&err, InputReaderError::Provider(m) if m.contains("expected 8, got 9")), + "got {err:?}" + ); + } + + #[test] + fn input_index_contiguity_rejects_a_truncated_tail_on_the_next_tick() { + // A tail truncation on a prior tick (input 8 missing) surfaces here: the + // next ingested input is 9 while the stored count is still 8. (The count + // witness catches the same truncation on the *same* tick — see below.) + let err = check_input_index_contiguity(8, &[9]) + .expect_err("a tail-truncated input must be caught on the next tick"); + assert!( + matches!(&err, InputReaderError::Provider(m) if m.contains("expected 8, got 9")), + "got {err:?}" + ); + } + + #[test] + fn input_count_complete_accepts_a_matching_count() { + // Stored 7, received 3 more → 10 total; chain agrees → complete. + assert!(check_input_count_complete(100, 10, 10).is_ok()); + // No new inputs and the chain has none beyond what we hold. + assert!(check_input_count_complete(100, 7, 7).is_ok()); + } + + #[test] + fn input_count_complete_rejects_a_truncated_tail_same_tick() { + // We received 7..=8 (received_total 9) but the chain has 10 inputs through + // this safe block — the tail (index 9) was dropped. Caught immediately, + // before the safe head is persisted. + let err = check_input_count_complete(100, 9, 10) + .expect_err("a truncated tail must fail the completeness witness"); + assert!( + matches!(&err, InputReaderError::Provider(m) + if m.contains("count at block 100 is 10, expected 9")), + "got {err:?}" + ); + } + + #[test] + fn input_count_complete_rejects_an_impossible_overcount() { + // We somehow hold more than the chain reports — also fail loud. + assert!(check_input_count_complete(100, 11, 10).is_err()); + } } diff --git a/sequencer/src/l1/submitter/mod.rs b/sequencer/src/l1/submitter/mod.rs index 7f53823..48178f8 100644 --- a/sequencer/src/l1/submitter/mod.rs +++ b/sequencer/src/l1/submitter/mod.rs @@ -4,9 +4,11 @@ //! Batch submitter: posts closed batches to L1 with at-least-once semantics. //! //! Each valid closed batch has a structural nonce (`batches.nonce`, set at -//! creation time as `parent.nonce + 1`). The scheduler checks that nonces are -//! strictly increasing and skips otherwise, so duplicates are deduplicated at -//! the scheduler level. See `worker` for the tick loop. +//! creation time as `parent.nonce + 1`). The scheduler accepts a batch only at +//! its exact expected nonce and rejects mismatches *without consuming the +//! nonce* (the staleness "skip" is a separate path) — at-least-once submission +//! is safe precisely because duplicates and replays arrive at already-consumed +//! nonces and are rejected. See `worker` for the tick loop. mod config; mod poster; diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 207d82a..9704afe 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -12,7 +12,8 @@ use sequencer_core::batch::Batch; use thiserror::Error; use tracing::{debug, info, warn}; -use crate::l1::partition::{decode_evm_advance_input, get_input_added_events}; +use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; +use crate::l1::watermark::WalletNonceWatermarkSink; pub type TxHash = alloy_primitives::B256; @@ -28,18 +29,33 @@ pub struct BatchPosterConfig { pub seconds_per_block: u64, /// Error codes that trigger `get_logs` retries with a shorter block range. pub long_block_range_error_codes: Vec, + /// The pinned deployment chain id. Re-confirmed against the RPC immediately + /// before every productive send (`submit_batches`), so a long-lived + /// submitter whose load-balanced RPC fails over to another chain refuses to + /// burn nonce slots on it rather than relying only on the one-shot boot / + /// reader checks. + pub expected_chain_id: u64, } #[derive(Debug, Error)] pub enum BatchPosterError { #[error("provider/transport: {0}")] Provider(String), + #[error("rpc chain id {rpc} does not match pinned chain id {expected}")] + ChainIdMismatch { rpc: u64, expected: u64 }, } #[async_trait] pub trait BatchPoster: Send + Sync { - async fn submit_batches(&self, payloads: Vec>) - -> Result, BatchPosterError>; + /// Broadcast the payloads as L1 txs at consecutive wallet nonces. + /// Implementations must raise `watermark` to the highest nonce they + /// are about to use *before* the first send (write-before-broadcast, + /// review R1a). + async fn submit_batches( + &self, + payloads: Vec>, + watermark: &dyn WalletNonceWatermarkSink, + ) -> Result, BatchPosterError>; async fn observed_submitted_batch_nonces( &self, @@ -158,17 +174,48 @@ impl BatchPoster for EthereumBatchPoster { async fn submit_batches( &self, payloads: Vec>, + watermark: &dyn WalletNonceWatermarkSink, ) -> Result, BatchPosterError> { if payloads.is_empty() { return Ok(Vec::new()); } + // Keyed-write chain-id gate (review): re-confirm the RPC still serves the + // pinned chain immediately before any productive send. The submitter is + // long-lived — its signing provider is built once at spawn and the + // boot-time / reader chain-id checks are one-shot — so a load-balanced + // RPC that fails over to another chain mid-life would otherwise burn + // submitter nonce slots on the wrong chain. Reached only when there is + // something to send (the empty early-return above), so idle ticks add no + // RPC load. A mismatch is terminal (lifted out of the transient bucket by + // the submitter run-loop); a transient RPC error retries like any blip. + let rpc_chain_id = self + .provider + .get_chain_id() + .await + .map_err(|err| BatchPosterError::Provider(err.to_string()))?; + if rpc_chain_id != self.config.expected_chain_id { + return Err(BatchPosterError::ChainIdMismatch { + rpc: rpc_chain_id, + expected: self.config.expected_chain_id, + }); + } + let fees = self .provider .estimate_eip1559_fees() .await .map_err(|err| BatchPosterError::Provider(err.to_string()))?; let mut next_nonce = self.latest_account_nonce().await?; + + // Write-before-broadcast (R1a): durably cover every nonce this + // tick will use before the first send. One raise to the highest + // covers the whole consecutive range. + let highest_nonce = next_nonce.saturating_add(payloads.len() as u64 - 1); + watermark + .raise_to(highest_nonce) + .map_err(BatchPosterError::Provider)?; + let mut tx_hashes = Vec::with_capacity(payloads.len()); for payload in payloads { @@ -202,7 +249,13 @@ impl BatchPoster for EthereumBatchPoster { return Ok(Vec::new()); } - let events = get_input_added_events( + // Ordered fetch: `advance_expected_batch_nonce` folds these nonces + // assuming L1 event order, so a raw `eth_getLogs` reorder would + // under-advance the frontier and resubmit an already-mined suffix + // (wasted gas + InputBox noise). The `_ordered` helper guarantees the + // canonical (block, tx_index, log_index) order — the same the reader + // relies on for its contiguity check. + let events = get_input_added_events_ordered( &self.provider, self.config.app_address, &self.config.l1_submit_address, @@ -239,6 +292,7 @@ impl BatchPoster for EthereumBatchPoster { #[cfg(test)] pub(crate) mod mock { use super::{Batch, BatchPoster, BatchPosterError, TxHash}; + use crate::l1::watermark::WalletNonceWatermarkSink; use async_trait::async_trait; use std::sync::Mutex; @@ -282,6 +336,7 @@ pub(crate) mod mock { async fn submit_batches( &self, payloads: Vec>, + _watermark: &dyn WalletNonceWatermarkSink, ) -> Result, BatchPosterError> { let mut tx_hashes = Vec::with_capacity(payloads.len()); for payload in payloads { @@ -322,9 +377,192 @@ pub(crate) mod mock { #[cfg(test)] mod tests { + use std::sync::Mutex; use std::time::Duration; - use super::{BatchPoster, derive_confirmation_timeout, mock::MockBatchPoster}; + use super::{ + BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, + derive_confirmation_timeout, mock::MockBatchPoster, + }; + use crate::l1::watermark::WalletNonceWatermarkSink; + use alloy::node_bindings::Anvil; + use alloy::providers::Provider; + use alloy::rpc::types::BlockNumberOrTag; + + fn require_anvil() { + assert!( + std::process::Command::new("anvil") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok(), + "anvil not found on PATH — install Foundry (https://getfoundry.sh)" + ); + } + + /// A watermark sink that records every `raise_to` call and (optionally) + /// fails, so a test can observe whether the raise happened — and in what + /// order relative to the first send. + struct RecordingWatermarkSink { + calls: Mutex>, + fail: bool, + } + + impl RecordingWatermarkSink { + fn failing() -> Self { + Self { + calls: Mutex::new(Vec::new()), + fail: true, + } + } + + fn passing() -> Self { + Self { + calls: Mutex::new(Vec::new()), + fail: false, + } + } + + fn calls(&self) -> Vec { + self.calls.lock().expect("lock").clone() + } + } + + impl WalletNonceWatermarkSink for RecordingWatermarkSink { + fn raise_to(&self, highest: u64) -> Result<(), String> { + self.calls.lock().expect("lock").push(highest); + if self.fail { + Err("recording sink: forced failure".to_string()) + } else { + Ok(()) + } + } + } + + /// R1a write-before-broadcast: `submit_batches` must raise the watermark to + /// cover the whole consecutive nonce range *before* the first send. We lock + /// it with a sink that fails on `raise_to`: a correct poster aborts the tick + /// before broadcasting anything, so the submitter's pending nonce is + /// unchanged. If `raise_to` were moved after the first `addInput` send + /// (re-opening the F1 zombie-tx hole), that send would bump the pending + /// nonce and this test would go red. Also pins the raise count (once) and + /// value (`base + payloads.len() - 1`). (Mutation-checked: moving the raise + /// after the send loop fails this test.) + #[tokio::test] + async fn submit_batches_raises_watermark_before_any_send() { + require_anvil(); + let anvil = Anvil::default().spawn(); + // Anvil account 0 — the submitter; its key signs the (never-sent) txs. + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key) + .expect("signer provider"); + + let config = BatchPosterConfig { + l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), + app_address: alloy_primitives::Address::repeat_byte(0x22), + batch_submitter_address: submitter, + start_block: 0, + confirmation_depth: 0, + seconds_per_block: 1, + long_block_range_error_codes: vec![], + expected_chain_id: anvil.chain_id(), + }; + let poster = EthereumBatchPoster::new(provider.clone(), config); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + let sink = RecordingWatermarkSink::failing(); + let payloads = vec![vec![0u8; 4], vec![1u8; 4], vec![2u8; 4]]; // 3 consecutive nonces + + let result = poster.submit_batches(payloads, &sink).await; + + assert!( + matches!(result, Err(BatchPosterError::Provider(_))), + "a failing watermark sink must abort submit_batches, got {result:?}" + ); + // (a) raised exactly once, (b) to the highest nonce of the range. + assert_eq!( + sink.calls(), + vec![base_nonce + 2], + "raise_to must be called once with base_nonce + payloads.len() - 1" + ); + // (c) before any send — no tx broadcast, so pending nonce is unchanged. + let pending = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .expect("pending nonce"); + assert_eq!( + pending, base_nonce, + "raise_to must run before any send; a broadcast would have bumped the pending nonce" + ); + } + + /// Keyed-write chain-id gate: a long-lived submitter pointed at an RPC that + /// serves a different chain than the pinned one must refuse to submit, before + /// any productive work — no watermark raise, no broadcast. Anvil's chain id + /// is 31337; we pin a different one and assert the `ChainIdMismatch` refusal + /// fires ahead of the (would-otherwise-fail-later) watermark raise. + #[tokio::test] + async fn submit_batches_refuses_on_wrong_chain_before_any_work() { + require_anvil(); + let anvil = Anvil::default().spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key) + .expect("signer provider"); + + let wrong_chain_id = anvil.chain_id() + 1; + let config = BatchPosterConfig { + l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), + app_address: alloy_primitives::Address::repeat_byte(0x22), + batch_submitter_address: submitter, + start_block: 0, + confirmation_depth: 0, + seconds_per_block: 1, + long_block_range_error_codes: vec![], + expected_chain_id: wrong_chain_id, + }; + let poster = EthereumBatchPoster::new(provider.clone(), config); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + // A sink that would *succeed* — so the only thing that can stop a send is + // the chain-id gate, not the watermark guard. (Recording proves the gate + // fires first: a passing chain check would reach `raise_to`.) + let sink = RecordingWatermarkSink::passing(); + let payloads = vec![vec![0u8; 4], vec![1u8; 4]]; + + let result = poster.submit_batches(payloads, &sink).await; + + assert!( + matches!( + result, + Err(BatchPosterError::ChainIdMismatch { rpc, expected }) + if rpc == anvil.chain_id() && expected == wrong_chain_id + ), + "wrong-chain RPC must abort submit_batches with ChainIdMismatch, got {result:?}" + ); + assert!( + sink.calls().is_empty(), + "chain-id gate must fire before the watermark raise (no raise_to call)" + ); + let pending = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .expect("pending nonce"); + assert_eq!( + pending, base_nonce, + "no tx may be broadcast on the wrong chain" + ); + } #[tokio::test] async fn mock_poster_tracks_requested_suffix_start_block() { diff --git a/sequencer/src/l1/submitter/worker.rs b/sequencer/src/l1/submitter/worker.rs index d47d885..5ea1763 100644 --- a/sequencer/src/l1/submitter/worker.rs +++ b/sequencer/src/l1/submitter/worker.rs @@ -78,7 +78,7 @@ fn decide_submit_start(frontier: SubmitterFrontier, recently_observed_nonces: &[ // unresolved nonce. The scan starts at `safe_block + 1` (the submitter // asks the poster for that), so wallet-nonce ordering guarantees the // observed list mirrors our submission order. - advance_expected_batch_nonce( + sequencer_core::protocol::advance_expected_batch_nonce( frontier.accepted_next_nonce, recently_observed_nonces.iter().copied(), ) @@ -88,12 +88,17 @@ pub struct BatchSubmitter { db_path: String, poster: Arc

, idle_poll_interval: Duration, + /// Write-before-broadcast hook (review R1a): the poster raises the + /// persisted wallet-nonce watermark through this before every send. + watermark_sink: crate::l1::watermark::StorageWatermarkSink, } impl BatchSubmitter

{ pub fn new(db_path: impl Into, poster: Arc

, config: BatchSubmitterConfig) -> Self { + let db_path = db_path.into(); Self { - db_path: db_path.into(), + watermark_sink: crate::l1::watermark::StorageWatermarkSink::new(db_path.clone()), + db_path, poster, idle_poll_interval: config.idle_poll_interval(), } @@ -137,6 +142,12 @@ impl BatchSubmitter

{ loop { let outcome = match self.tick_once().await { Ok(o) => o, + // A wrong-chain RPC is terminal — never retry-loop signing onto + // it. Lift it out of the transient `Poster` bucket below. + Err(e @ BatchSubmitterError::Poster(BatchPosterError::ChainIdMismatch { .. })) => { + error!(error = %e, "RPC serves the wrong chain — refusing to submit"); + return Err(e); + } Err(BatchSubmitterError::Poster(source)) => { error!(error = %source, "L1 provider error — will retry"); TickOutcome::Transient @@ -180,7 +191,10 @@ impl BatchSubmitter

{ } let submitted_count = pending.len(); let payloads: Vec> = pending.into_iter().map(|b| b.encoded).collect(); - let tx_hashes = self.poster.submit_batches(payloads).await?; + let tx_hashes = self + .poster + .submit_batches(payloads, &self.watermark_sink) + .await?; if tx_hashes.len() != submitted_count { return Err(BatchSubmitterError::Poster(BatchPosterError::Provider( format!( @@ -221,38 +235,6 @@ impl BatchSubmitter

{ } } -/// Advance `expected` by greedily consuming any matching observed nonce. -/// -/// `observed_nonces` is the stream of **batch nonces** (from the SSZ payload) -/// decoded from `InputAdded` events sent by our batch-submitter EOA, in L1 -/// event order. Because L1 mines txs from a single EOA in strict wallet-nonce -/// order, this stream is naturally gap-less at the wallet-nonce level: -/// tx[k]'s event cannot appear on-chain without tx[k-1]'s event, and the -/// observed batch nonce sequence therefore mirrors our submission order. -/// -/// Batch nonces themselves (unlike wallet nonces) CAN repeat across recovery -/// generations — e.g., after a cascade, a fresh batch reuses its invalidated -/// predecessor's nonce. That's why we still match on equality rather than -/// trusting a sort: in a post-recovery window, the same batch nonce can be -/// observed twice (once from the invalidated generation, once from the new -/// one), and we only want to advance once. -/// -/// Under the wallet-nonce ordering above, once the next `expected` doesn't -/// appear in the stream the frontier naturally stops advancing — the gap -/// means the scheduler hasn't seen that nonce on-chain yet (or observed it at -/// a different wallet nonce from an earlier generation). -fn advance_expected_batch_nonce( - mut expected: u64, - observed_nonces: impl IntoIterator, -) -> u64 { - for nonce in observed_nonces { - if nonce == expected { - expected = expected.saturating_add(1); - } - } - expected -} - #[cfg(test)] mod tests { use std::sync::Arc; @@ -306,14 +288,13 @@ mod tests { fn seed_safe_submitted_batches(db_path: &str, safe_block: u64, nonces: &[u64]) { let mut storage = Storage::open(db_path).expect("open storage"); + // Landings carry the local batch's real wire bytes so the + // content-identity check (review R2) accepts them. let inputs: Vec<_> = nonces .iter() .map(|nonce| StoredSafeInput { sender: BATCH_SUBMITTER_ADDRESS, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: *nonce, - frames: Vec::new(), - }), + payload: crate::storage::test_helpers::local_batch_payload(&mut storage, *nonce), block_number: safe_block, }) .collect(); @@ -495,19 +476,4 @@ mod tests { // expected=3, skip. 3 matches → advance to 4. assert_eq!(from_nonce, 4); } - - #[test] - fn advance_expected_batch_nonce_matches_scheduler_nonce_rule() { - assert_eq!(super::advance_expected_batch_nonce(0, Vec::::new()), 0); - assert_eq!(super::advance_expected_batch_nonce(0, vec![0, 1, 2]), 3); - assert_eq!(super::advance_expected_batch_nonce(0, vec![0, 2, 3]), 1); - assert_eq!(super::advance_expected_batch_nonce(0, vec![1, 2, 3]), 0); - assert_eq!(super::advance_expected_batch_nonce(0, vec![0, 1, 1, 2]), 3); - assert_eq!( - super::advance_expected_batch_nonce(0, vec![6, 4, 3, 2, 2, 0, 1]), - 2 - ); - assert_eq!(super::advance_expected_batch_nonce(0, vec![0, 2, 1]), 2); - assert_eq!(super::advance_expected_batch_nonce(2, vec![2, 3]), 4); - } } diff --git a/sequencer/src/l1/watermark.rs b/sequencer/src/l1/watermark.rs new file mode 100644 index 0000000..18fef18 --- /dev/null +++ b/sequencer/src/l1/watermark.rs @@ -0,0 +1,50 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Write-before-broadcast hook for the wallet-nonce watermark (review R1a). +//! +//! Every component that broadcasts a transaction from the batch-submitter key +//! — the batch poster and the mempool flusher's no-ops alike — must first +//! durably commit `watermark = max(watermark, highest_nonce_about_to_send)` +//! and only then send. One uniform rule, no case analysis: the invariant is +//! simply "the watermark covers the nonce of everything we ever sent", which +//! is what lets the flush consume every slot we ever used without trusting +//! the local node's volatile mempool memory (the F1 zombie counterexample). +//! +//! A crash between the commit and the send only over-covers: the flush later +//! no-ops a never-used slot — one wasted no-op, harmless. + +use crate::storage::Storage; + +/// Durable raise of the wallet-nonce watermark, called *before* broadcasting. +/// `raise_to(h)` must commit `watermark = max(watermark, h)` power-loss +/// durably before returning; the caller may then broadcast txs at nonces +/// `<= h`. +pub trait WalletNonceWatermarkSink: Send + Sync { + fn raise_to(&self, highest: u64) -> Result<(), String>; +} + +/// Sink backed by the sequencer DB's `wallet_nonce_watermark` singleton. +/// Opens a short-lived writer connection per raise — raises happen at most +/// once per submitter tick / flush pass, well off the hot path. +pub struct StorageWatermarkSink { + db_path: String, +} + +impl StorageWatermarkSink { + pub fn new(db_path: impl Into) -> Self { + Self { + db_path: db_path.into(), + } + } +} + +impl WalletNonceWatermarkSink for StorageWatermarkSink { + fn raise_to(&self, highest: u64) -> Result<(), String> { + let mut storage = Storage::open_writer(&self.db_path) + .map_err(|e| format!("watermark sink: open storage: {e}"))?; + storage + .raise_wallet_nonce_watermark(highest) + .map_err(|e| format!("watermark sink: raise: {e}")) + } +} diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index a40c98d..120a99a 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -17,6 +17,7 @@ //! invariant the storage layer relies on. pub mod egress; +pub mod harness; pub mod http; pub mod ingress; pub mod l1; @@ -24,6 +25,7 @@ pub mod recovery; pub mod runtime; pub mod storage; +pub use harness::{Cli, Command, dispatch, run_main}; pub use http::{ApiConfig, ApiError, WS_CATCHUP_WINDOW_EXCEEDED_REASON}; -pub use runtime::config::RunConfig; +pub use runtime::config::{FlushConfig, RunConfig, SetupConfig}; pub use runtime::{RunError, run}; diff --git a/sequencer/src/recovery/detector.rs b/sequencer/src/recovery/detector.rs index c9a5f11..1bd7c18 100644 --- a/sequencer/src/recovery/detector.rs +++ b/sequencer/src/recovery/detector.rs @@ -162,17 +162,6 @@ mod tests { } } - fn make_stale_batch_payload(nonce: u64, safe_block: u64) -> Vec { - ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce, - frames: vec![sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block, - fee_price: 0, - }], - }) - } - #[tokio::test] async fn exits_on_shutdown_when_safe() { let db = temp_db("detector-shutdown"); @@ -217,12 +206,13 @@ mod tests { .expect("close batch 1"); let protocol = test_protocol(); + let landed = crate::storage::test_helpers::local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1135, &[StoredSafeInput { sender: SENDER_A, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -272,12 +262,13 @@ mod tests { .expect("close batch 1"); let protocol = test_protocol(); + let landed = crate::storage::test_helpers::local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1200, &[StoredSafeInput { sender: SENDER_A, - payload: make_stale_batch_payload(0, 100), + payload: landed, block_number: 200, }], SENDER_A, diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index 308086a..e2207af 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -19,6 +19,8 @@ use std::time::Duration; use thiserror::Error; use tracing::{debug, error, info}; +use crate::l1::watermark::{StorageWatermarkSink, WalletNonceWatermarkSink}; + #[derive(Debug, Error)] pub enum FlushError { #[error("provider/transport: {0}")] @@ -92,6 +94,28 @@ fn map_watch_error(err: PendingTransactionError) -> Result { } impl MempoolFlusher { + /// Build a flusher over `provider` plus the DB-backed watermark sink, run + /// the watermark-anchored flush, and return the observed safe block `C`. + /// + /// This is the flush-acquire core shared by all three keyed-flush sites + /// (`setup --recovery`, the runtime danger path, and `flush-mempool`). They + /// differ only in where the signing key, submitter address, and watermark + /// come from, and in the surrounding error type — so callers resolve those + /// (provider creation keeps each site's own error mapping; the returned + /// [`FlushError`] maps into `RunError`/`RecoveryError` via the existing + /// `From` impls) and this owns the sink build + `flush_and_wait`. + pub(crate) async fn flush_to_safe( + provider: DynProvider, + submitter_address: Address, + seconds_per_block: u64, + db_path: impl Into, + watermark: Option, + ) -> Result { + let flusher = Self::new(provider, submitter_address, seconds_per_block); + let sink = StorageWatermarkSink::new(db_path); + flusher.flush_and_wait(watermark, &sink).await + } + pub fn new(provider: DynProvider, address: Address, seconds_per_block: u64) -> Self { let (confirmation_timeout, safe_poll_interval) = derive_timeouts(seconds_per_block); Self { @@ -102,40 +126,73 @@ impl MempoolFlusher { } } - /// Flush the mempool by submitting no-op transactions for pending nonce - /// slots, then waiting until every slot is safe. + /// Flush the mempool by submitting no-op transactions for unresolved + /// nonce slots, then waiting until every slot we ever used is safe. + /// + /// `watermark` is the persisted wallet-nonce watermark — the highest + /// nonce this deployment ever broadcast (review R1a), or `None` if + /// nothing was ever broadcast (or no DB survives, the cockroach-recovery + /// best-effort case, R1b). The loop runs until + /// + /// ```text + /// pending <= safe && safe >= watermark + 1 + /// ``` /// - /// The loop runs until `get_transaction_count(Pending) <= get_transaction_count(Safe)`, - /// meaning every slot has reached safe finality. + /// The first conjunct resolves every slot the local node remembers; the + /// second is the durable anchor — it refuses to declare victory until + /// slot `watermark` is consumed at safe depth, covering zombie txs the + /// local node has forgotten but the network may still hold (the F1 + /// counterexample). It doubles as the post-flush assert from R1a: the + /// function cannot return success without it. /// /// At each iteration: - /// 1. Submit 0-ETH self-transfers for nonces between `Latest` and `Pending`. - /// These compete with any batch transactions still in the mempool. If - /// an original batch wins, that is also success: the slot advanced. + /// 1. Submit 0-ETH self-transfers for nonces in + /// `[Latest, max(Pending, watermark + 1))` — every slot not yet + /// mined that we might ever have used. The sink raises the persisted + /// watermark before the broadcast (uniform write-before-broadcast, + /// though no-ops never exceed the existing watermark under the + /// invariant). These compete with any of our txs still in the + /// network; whichever wins, the slot advances. /// 2. Watch each submitted no-op for L1 inclusion. /// 3. Sleep to let the safe head advance, then re-check the loop condition. /// 4. If any watch times out, retry the outer loop (tx may have been dropped, /// or the original batch may be making progress instead). - pub async fn flush_and_wait(&self) -> Result<(), FlushError> { + /// + /// Returns the L1 **safe block number** at which resolution was observed + /// (review F2): the caller must not cascade until its own re-synced view + /// reaches at least this block. + pub async fn flush_and_wait( + &self, + watermark: Option, + sink: &dyn WalletNonceWatermarkSink, + ) -> Result { let mut attempt = 0u32; loop { let safe_nonce = self.nonce_at(BlockNumberOrTag::Safe).await?; let pending_nonce = self.nonce_at(BlockNumberOrTag::Pending).await?; + // The durable anchor: every slot we ever used must be consumed + // at safe depth, regardless of what the local pool remembers. + let required_safe_nonce = watermark.map_or(0, |w| w.saturating_add(1)); - if pending_nonce <= safe_nonce { + if pending_nonce <= safe_nonce && safe_nonce >= required_safe_nonce { + let safe_block = self.safe_block_number().await?; info!( safe_nonce, + required_safe_nonce, + safe_block, "mempool flush complete — all slots reached safe finality" ); - return Ok(()); + return Ok(safe_block); } - let unresolved = pending_nonce - safe_nonce; + let flush_end = pending_nonce.max(required_safe_nonce); + let unresolved = flush_end.saturating_sub(safe_nonce); if attempt == 0 { info!( safe_nonce, pending_nonce, + required_safe_nonce, unresolved, "flushing mempool: submitting no-ops for unresolved w_nonce slots" ); @@ -146,17 +203,26 @@ impl MempoolFlusher { attempt, safe_nonce, pending_nonce, + required_safe_nonce, unresolved, "flush retry: previous attempt timed out, resubmitting" ); } attempt += 1; - // Submit no-ops for nonces between Latest and Pending. We submit - // the full range before watching any tx, so every unresolved slot - // gets a competing no-op attempt in this pass. + // Submit no-ops for every not-yet-mined slot up to the flush end. + // The full range goes out before watching any tx, so every + // unresolved slot gets a competing no-op attempt in this pass. let latest_nonce = self.nonce_at(BlockNumberOrTag::Latest).await?; - let tx_hashes = self.submit_noops(latest_nonce, pending_nonce).await?; + if latest_nonce < flush_end { + // Uniform write-before-broadcast; covers the no-ops we are + // about to send (a no-op above the current watermark can only + // happen if someone else used our key — over-covering then is + // exactly right). + sink.raise_to(flush_end.saturating_sub(1)) + .map_err(FlushError::Provider)?; + } + let tx_hashes = self.submit_noops(latest_nonce, flush_end).await?; // Watch each submitted tx for L1 inclusion. if !self.watch_txs(&tx_hashes).await? { @@ -168,6 +234,17 @@ impl MempoolFlusher { } } + /// Block number of the current L1 safe head. + async fn safe_block_number(&self) -> Result { + let block = self + .provider + .get_block_by_number(BlockNumberOrTag::Safe) + .await + .map_err(|e| FlushError::Provider(e.to_string()))? + .ok_or_else(|| FlushError::Provider("no safe block available".to_string()))?; + Ok(block.header.number) + } + /// Submit 0-ETH self-transfers for nonces `from_nonce..to_nonce`. /// Returns the tx hashes of successfully submitted transactions. async fn submit_noops(&self, from_nonce: u64, to_nonce: u64) -> Result, FlushError> { @@ -283,6 +360,14 @@ mod tests { use alloy::node_bindings::Anvil; use alloy::providers::Provider; + /// Sink for tests that don't assert on watermark raises. + struct NoopWatermarkSink; + impl WalletNonceWatermarkSink for NoopWatermarkSink { + fn raise_to(&self, _highest: u64) -> Result<(), String> { + Ok(()) + } + } + // ── H5: replacement-fee bump keeps no-ops competitive ───────── #[test] @@ -469,7 +554,10 @@ mod tests { let flusher = MempoolFlusher::new(provider, addr, 12); // No pending txs — should return immediately. - flusher.flush_and_wait().await.expect("flush"); + flusher + .flush_and_wait(None, &NoopWatermarkSink) + .await + .expect("flush"); } #[tokio::test] @@ -506,10 +594,13 @@ mod tests { // Run the flusher — it should resolve all 3 nonces to safe. let flusher = MempoolFlusher::new(provider.clone(), addr, 12) .with_timeouts(Duration::from_secs(5), Duration::from_millis(200)); - tokio::time::timeout(Duration::from_secs(10), flusher.flush_and_wait()) - .await - .expect("flush should complete within timeout") - .expect("flush should succeed"); + tokio::time::timeout( + Duration::from_secs(10), + flusher.flush_and_wait(None, &NoopWatermarkSink), + ) + .await + .expect("flush should complete within timeout") + .expect("flush should succeed"); // Verify: safe nonce caught up. let safe_after = provider @@ -523,6 +614,46 @@ mod tests { ); } + #[tokio::test] + async fn flush_covers_watermark_slots_the_pool_forgot() { + require_anvil(); + + let anvil = spawn_anvil(); + let provider = signer_provider(&anvil); + let addr = anvil.addresses()[0]; + + // Models the F1 zombie: the persisted watermark says slot 0 was + // broadcast, but the local pool has no memory of it + // (pending == safe == 0). The pre-R1a `pending <= safe` early + // return would declare victory immediately and leave the slot to + // a zombie; the anchored flush must consume slot 0 with a no-op + // and wait for it to reach safe depth. + let _miner = start_miner(provider.clone(), Duration::from_millis(100)); + let flusher = MempoolFlusher::new(provider.clone(), addr, 12) + .with_timeouts(Duration::from_secs(5), Duration::from_millis(200)); + let observed_safe_block = tokio::time::timeout( + Duration::from_secs(10), + flusher.flush_and_wait(Some(0), &NoopWatermarkSink), + ) + .await + .expect("anchored flush should complete within timeout") + .expect("anchored flush should succeed"); + + let safe_after = provider + .get_transaction_count(addr) + .block_id(BlockNumberOrTag::Safe.into()) + .await + .expect("safe nonce after flush"); + assert!( + safe_after >= 1, + "watermark slot 0 must be consumed at safe depth, got {safe_after}" + ); + assert!( + observed_safe_block > 0, + "flush must report the safe block it observed resolution at (F2)" + ); + } + #[tokio::test] async fn flush_handles_already_mined_but_not_safe() { require_anvil(); @@ -560,10 +691,13 @@ mod tests { // Flusher should wait for safe finality (no new txs to submit). let flusher = MempoolFlusher::new(provider.clone(), addr, 12) .with_timeouts(Duration::from_secs(5), Duration::from_millis(200)); - tokio::time::timeout(Duration::from_secs(10), flusher.flush_and_wait()) - .await - .expect("flush should complete within timeout") - .expect("flush should succeed"); + tokio::time::timeout( + Duration::from_secs(10), + flusher.flush_and_wait(None, &NoopWatermarkSink), + ) + .await + .expect("flush should complete within timeout") + .expect("flush should succeed"); let safe_after = provider .get_transaction_count(addr) @@ -638,10 +772,13 @@ mod tests { // `flush_and_wait` must fail fast (no internal retry loop). Wrap in // a generous outer timeout just to bound test flakiness if alloy's // HTTP client has small internal retries. - let err = tokio::time::timeout(Duration::from_secs(5), flusher.flush_and_wait()) - .await - .expect("flush_and_wait must not hang under disconnect") - .expect_err("flush_and_wait must surface a Provider error under disconnect"); + let err = tokio::time::timeout( + Duration::from_secs(5), + flusher.flush_and_wait(None, &NoopWatermarkSink), + ) + .await + .expect("flush_and_wait must not hang under disconnect") + .expect_err("flush_and_wait must surface a Provider error under disconnect"); assert!( matches!(err, FlushError::Provider(_)), "expected FlushError::Provider, got: {err:?}", @@ -658,10 +795,13 @@ mod tests { // fee no-op (or let the original land), wait for safe, and return. let flusher_after = MempoolFlusher::new(proxied_provider, addr, 12) .with_timeouts(Duration::from_secs(5), Duration::from_millis(200)); - tokio::time::timeout(Duration::from_secs(15), flusher_after.flush_and_wait()) - .await - .expect("flush_and_wait should complete after reconnect") - .expect("flush should succeed once the provider is reachable"); + tokio::time::timeout( + Duration::from_secs(15), + flusher_after.flush_and_wait(None, &NoopWatermarkSink), + ) + .await + .expect("flush_and_wait should complete after reconnect") + .expect("flush should succeed once the provider is reachable"); // Forward progress: the nonce-0 slot was consumed (either by the // flusher's no-op or by the original tx landing). `safe_nonce` is diff --git a/sequencer/src/recovery/mod.rs b/sequencer/src/recovery/mod.rs index 8350dcc..c7a4139 100644 --- a/sequencer/src/recovery/mod.rs +++ b/sequencer/src/recovery/mod.rs @@ -50,7 +50,7 @@ use crate::l1::reader::{InputReader, InputReaderError}; use crate::runtime::config::L1Config; use crate::storage::{self, DangerStatus, StorageOpenError}; pub use detector::{DangerDetector, DangerDetectorError, DetectorExit}; -pub use flusher::MempoolFlusher; +pub use flusher::{FlushError, MempoolFlusher}; use sequencer_core::protocol::ProtocolTiming; #[derive(Debug, Error)] @@ -65,8 +65,38 @@ pub enum RecoveryError { InputReader(#[from] InputReaderError), #[error("provider: {0}")] Provider(String), + #[error("recovery flush chain-id mismatch: rpc {rpc} != pinned {expected}")] + ChainIdMismatch { rpc: u64, expected: u64 }, #[error("startup refused: {0:?}")] Refuse(RefuseReason), + #[error( + "post-flush re-sync reached safe block {resynced_safe_block}, behind the \ + flusher's observed resolution at {flush_observed_safe_block}; refusing to \ + cascade on a lagging L1 view (respawn retries with a fresher view)" + )] + ResyncBehindFlushView { + resynced_safe_block: u64, + flush_observed_safe_block: u64, + }, +} + +/// F2 coherence guard: refuse if the post-flush re-sync's safe head lags the +/// block the flusher observed resolution at. Folding (`setup --recovery`) or +/// cascading (runtime danger) on a view that stops short of the flush's +/// resolution would miss inputs the flush already settled. Shared by both +/// recovery paths; the orchestrator respawn retries with a fresher L1 view (the +/// flush is idempotent). See [`RecoveryError::ResyncBehindFlushView`]. +pub(crate) fn assert_resync_caught_up( + resynced_safe_block: u64, + flush_observed_safe_block: u64, +) -> Result<(), RecoveryError> { + if resynced_safe_block < flush_observed_safe_block { + return Err(RecoveryError::ResyncBehindFlushView { + resynced_safe_block, + flush_observed_safe_block, + }); + } + Ok(()) } /// Why startup cannot proceed safely. @@ -75,6 +105,13 @@ pub enum RecoveryError { /// startup unsafe. The operator sees the variant in logs and must intervene. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RefuseReason { + /// A fully-accepted L1 landing failed the content-identity check + /// (review R2): canonical state diverged from the local batch tree. + /// Terminal — no restart self-heals it; the operator must run cockroach + /// recovery (wipe + rebuild from L1). Standard recovery is forbidden: + /// it reconciles the tree's *shape* assuming accepted nonce N is our + /// batch N, which is exactly what no longer holds. + CanonicalDivergence { nonce: u64 }, /// The L1 safe block timestamp is too old or unknown, so the local L1 view /// is not usable for recovery or continued soft confirmations. L1ViewStale, @@ -124,29 +161,15 @@ impl StartupAction { } } -fn danger_status_label(danger: DangerStatus) -> &'static str { - match danger { - DangerStatus::Safe => "safe", - DangerStatus::L1ViewStale => "l1_view_stale", - DangerStatus::ClosedBatchInDanger(_) => "closed_batch_in_danger", - DangerStatus::TipInDanger(_) => "tip_in_danger", - DangerStatus::EstimatedBatchInDanger(_) => "estimated_batch_in_danger", - } -} - -fn danger_batch_index(danger: DangerStatus) -> Option { - match danger { - DangerStatus::ClosedBatchInDanger(batch_index) - | DangerStatus::TipInDanger(batch_index) - | DangerStatus::EstimatedBatchInDanger(batch_index) => Some(batch_index), - DangerStatus::Safe | DangerStatus::L1ViewStale => None, - } -} - -fn refuse_reason_label(reason: RefuseReason) -> &'static str { - match reason { - RefuseReason::L1ViewStale => "l1_view_stale", - RefuseReason::EstimatedBatchInDanger { .. } => "estimated_batch_in_danger", +impl RefuseReason { + /// Stable label for logs/metrics. Inherent method, co-located with the + /// variants (`DangerStatus::label`/`batch_index` live next to that enum). + fn label(self) -> &'static str { + match self { + RefuseReason::CanonicalDivergence { .. } => "canonical_divergence", + RefuseReason::L1ViewStale => "l1_view_stale", + RefuseReason::EstimatedBatchInDanger { .. } => "estimated_batch_in_danger", + } } } @@ -156,6 +179,9 @@ fn refuse_reason_label(reason: RefuseReason) -> &'static str { pub fn decide_startup_action(danger: DangerStatus) -> StartupAction { match danger { DangerStatus::Safe => StartupAction::Proceed, + DangerStatus::CanonicalDivergence(nonce) => { + StartupAction::Refuse(RefuseReason::CanonicalDivergence { nonce }) + } DangerStatus::ClosedBatchInDanger(batch_index) => { StartupAction::FlushAndCascade { batch_index } } @@ -213,8 +239,8 @@ pub async fn run_preemptive_recovery( }; let action = decide_startup_action(danger); tracing::info!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), l1_reachable, danger_threshold = protocol.danger_threshold(), @@ -242,8 +268,8 @@ pub async fn run_preemptive_recovery( let invalidated = match action { StartupAction::Proceed => { tracing::info!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), "no danger zone detected — proceeding without recovery" ); @@ -257,8 +283,8 @@ pub async fn run_preemptive_recovery( } StartupAction::RecoverTip { batch_index } => { tracing::error!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), tip_batch_index = batch_index, danger_threshold = protocol.danger_threshold(), @@ -269,8 +295,8 @@ pub async fn run_preemptive_recovery( } StartupAction::FlushAndCascade { batch_index } => { tracing::error!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), batch_index, danger_threshold = protocol.danger_threshold(), @@ -281,11 +307,11 @@ pub async fn run_preemptive_recovery( } StartupAction::Refuse(reason) => { tracing::error!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), ?reason, - refuse_reason = refuse_reason_label(reason), + refuse_reason = reason.label(), l1_reachable, "startup refused: cannot recover safely" ); @@ -295,8 +321,8 @@ pub async fn run_preemptive_recovery( if invalidated.is_empty() { tracing::info!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), invalidated_count = 0, "startup recovery complete — no batches invalidated" @@ -307,8 +333,8 @@ pub async fn run_preemptive_recovery( // log already alerted the operator at error level; this completes // that incident with a non-error outcome. tracing::warn!( - danger_status = danger_status_label(danger), - danger_batch_index = ?danger_batch_index(danger), + danger_status = danger.label(), + danger_batch_index = ?danger.batch_index(), startup_action = action.label(), invalidated_count = invalidated.len(), batches = ?invalidated, @@ -332,17 +358,41 @@ async fn run_flush_and_cascade( l1_config: &L1Config, protocol: &ProtocolTiming, ) -> Result, RecoveryError> { - let flush_provider = crate::l1::provider::create_signer_provider( + // Keyed-write chain-id gate (review): the flush signs L1 no-op txs, so it + // must confirm the RPC still serves the pinned chain *immediately before + // signing*. The boot-time `validate_rpc_chain_id` and the reader's one-shot + // `verify_chain_id` are both stale by now (a load-balanced RPC could have + // failed over to another chain since), so neither is a sufficient backstop + // for a fresh keyed write. `create_verified_signer_provider` folds the check + // into the signer build so this path cannot skip it. A mismatch is terminal + // (operator misconfig); an RPC error is retryable (handled like `Provider`). + let flush_provider = crate::l1::provider::create_verified_signer_provider( &l1_config.eth_rpc_url, &l1_config.batch_submitter_private_key, + l1_config.chain_id, ) - .map_err(|e| RecoveryError::Provider(e.to_string()))?; - let flusher = MempoolFlusher::new( + .await + .map_err(|e| match e { + crate::l1::provider::VerifiedSignerProviderError::ChainIdMismatch { rpc, expected } => { + RecoveryError::ChainIdMismatch { rpc, expected } + } + other => RecoveryError::Provider(other.to_string()), + })?; + // The persisted watermark anchors the flush: every slot this deployment + // ever broadcast must resolve at safe depth, regardless of what the + // local node's pool remembers (review R1a / F1). + let watermark = { + let mut storage = storage::Storage::open(db_path)?; + storage.wallet_nonce_watermark()? + }; + let flush_observed_safe_block = MempoolFlusher::flush_to_safe( flush_provider, l1_config.batch_submitter_address, protocol.seconds_per_block, - ); - flusher.flush_and_wait().await?; + db_path, + watermark, + ) + .await?; // If this re-sync errors out, L1 has been flushed but the DB has NOT been // cascaded — we exit with the InputReaderError and rely on the orchestrator @@ -351,10 +401,13 @@ async fn run_flush_and_cascade( // - `flush_and_wait` is idempotent: on the next attempt it queries L1 for // pending wallet-nonces, finds zero (the previous flush cleared them), // and returns immediately. - // - `check_danger` is stable across the failure window: safe_block only - // moves forward and flush doesn't retroactively change closed batches' - // `first_frame_safe_block`, so the danger condition that fired before - // still fires after the restart. + // - `check_danger` re-decides on the post-flush state. Two cases: the + // danger persists (the restart re-enters this same path — flush is a + // no-op the second time), or the original danger resolved during the + // flush (e.g. the frontier batch landed gold), in which case the + // restart proceeds normally with any no-op'd Pending batch left valid — + // safe, since it simply resubmits at a fresh slot with no poisoned + // ancestor. // - `recover_post_flush` is idempotent against the resulting DB state // (verified by `after_post_recovery_crash_is_no_op` in `recovery_tests`). // @@ -368,6 +421,17 @@ async fn run_flush_and_cascade( tracing::info!("running post-flush recovery (cascade non-gold suffix)"); let mut storage = storage::Storage::open(db_path)?; + + // Coherence check (review F2): the cascade's precondition is that the + // gold frontier reflects at least the safe view the flusher observed + // resolution at. Behind a load-balanced RPC, the reader's re-sync can be + // served by a replica lagging the flusher's view — cascading then could + // invalidate a batch the scheduler actually accepted and reuse its + // nonce. Refuse instead; the orchestrator respawn retries with a + // fresher view (the flush is idempotent). + let resynced_safe_block = storage.current_safe_block()?.unwrap_or(0); + assert_resync_caught_up(resynced_safe_block, flush_observed_safe_block)?; + Ok(storage.recover_post_flush(protocol.danger_threshold())?) } @@ -383,6 +447,17 @@ mod tests { ); } + #[test] + fn refuse_on_canonical_divergence() { + // Terminal refusal — never `Proceed`, never a flush+cascade on top + // of a diverged frontier (review R2). The remedy is cockroach + // recovery, outside this dispatch entirely. + assert_eq!( + decide_startup_action(DangerStatus::CanonicalDivergence(7)), + StartupAction::Refuse(RefuseReason::CanonicalDivergence { nonce: 7 }) + ); + } + #[test] fn flush_and_cascade_on_closed_batch_in_danger() { assert_eq!( diff --git a/sequencer/src/runtime/config.rs b/sequencer/src/runtime/config.rs index 8fc55b2..52df617 100644 --- a/sequencer/src/runtime/config.rs +++ b/sequencer/src/runtime/config.rs @@ -1,9 +1,22 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +//! CLI configuration for the three subcommands (`setup`, `run`, +//! `flush-mempool`). +//! +//! The phase split: `setup` is L1-read-only and establishes the +//! timeless deployment identity + initial sync + genesis snapshot; it takes +//! the batch-submitter **address** but never the signing key. `run` boots +//! from an already-set-up DB, reads identity from the DB (so chain id / app +//! address are not CLI args here), and keeps the signing **key** because it +//! submits. `flush-mempool` is a keyed operator tool that settles the wallet +//! nonce on demand. +//! +//! Shared, drift-prone arg groups (`TimingArgs`, `KeyArgs`) are flattened into +//! the per-subcommand configs so defaults and env-var names stay consistent. + use alloy_primitives::Address; -use alloy_sol_types::Eip712Domain; -use clap::{ArgGroup, Parser}; +use clap::{ArgGroup, Args}; use sequencer_core::protocol::{ProtocolTiming, ProtocolTimingError}; const DEFAULT_HTTP_ADDR: &str = "127.0.0.1:3000"; @@ -12,9 +25,9 @@ const DB_FILENAME: &str = "sequencer.db"; /// Shared L1 / InputBox configuration used by both the input reader and the batch submitter. /// -/// Built once at startup from `RunConfig` plus the discovered InputBox address, so RPC URL, -/// InputBox address, and app address are defined in a single place and not duplicated across -/// component configs. +/// Built once at startup from the pinned deployment identity plus the runtime +/// `RunConfig`, so RPC URL, InputBox address, and app address are defined in a +/// single place and not duplicated across component configs. #[derive(Debug, Clone)] pub struct L1Config { pub eth_rpc_url: String, @@ -22,51 +35,78 @@ pub struct L1Config { pub app_address: Address, pub batch_submitter_private_key: String, pub batch_submitter_address: Address, + /// The pinned deployment chain id. Carried here so keyed-write paths (e.g. + /// the preemptive-recovery flush) can re-confirm the RPC's chain id right + /// before signing via [`crate::l1::provider::create_verified_signer_provider`]. + pub chain_id: u64, } -#[derive(Debug, Clone, Parser)] -#[command( - name = "sequencer", - about = "Deterministic sequencer prototype with low-latency soft confirmations.\n\n\ - All options can also be set via environment variables (shown in brackets).", - version, - after_help = "\ -Examples: - sequencer \\ - --eth-rpc-url http://127.0.0.1:8545 \\ - --chain-id 31337 \\ - --app-address 0x1111111111111111111111111111111111111111 \\ - --batch-submitter-private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 - - CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \\ - CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \\ - CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \\ - CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \\ - sequencer\ -", - group( - ArgGroup::new("batch_submitter_key_source") - .args(&["batch_submitter_private_key", "batch_submitter_private_key_file"]) - .required(true) - .multiple(false) - ) -)] -pub struct RunConfig { - #[arg(long, env = "CARTESI_SEQUENCER_HTTP_ADDR", default_value = DEFAULT_HTTP_ADDR, value_parser = parse_non_empty_string)] - pub http_addr: String, - #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] - pub data_dir: String, - #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] - pub eth_rpc_url: String, - /// Error codes that trigger `get_logs` retries with a shorter block range. - #[arg(long, env = "CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES", value_delimiter = ',', default_values = crate::l1::partition::DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES)] - pub long_block_range_error_codes: Vec, - /// Expected chain ID. Validated against the RPC at startup. - #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_ID")] - pub chain_id: u64, - /// Application (EIP-712 verifying contract) address. - #[arg(long, env = "CARTESI_SEQUENCER_APP_ADDRESS", value_parser = parse_address)] - pub app_address: Address, +/// Full path to the SQLite database file inside `data_dir`. +pub fn db_path_in(data_dir: &str) -> String { + std::path::Path::new(data_dir) + .join(DB_FILENAME) + .to_string_lossy() + .into_owned() +} + +/// Protocol-timing tuning knobs shared by `setup` and `run`. `setup` needs a +/// valid `ProtocolTiming` for the initial-sync writes (and validating it here +/// fails fast at setup, not just at run). +#[derive(Debug, Clone, Args)] +pub struct TimingArgs { + /// Blocks before MAX_WAIT_BLOCKS to trigger preemptive recovery. + /// The danger threshold is MAX_WAIT_BLOCKS minus this margin. + /// Must be less than MAX_WAIT_BLOCKS (validated at startup). + /// + /// Default 300 (~1h at 12s/block) is sized to give operators meaningful + /// runway to investigate before the system gives up on the current + /// batches — see `docs/recovery/README.md` "Step 1: Danger threshold". + #[arg( + long, + env = "CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS", + default_value = "300" + )] + pub preemptive_margin_blocks: u64, + + /// Blocks of safe-head age after which the L1 read view is considered too + /// stale to trust. Independent of the preemptive margin. Must be strictly + /// less than the danger threshold (validated at startup). + /// + /// Default 600 (~2h at 12s/block). + #[arg(long, env = "CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS", default_value = "600", value_parser = clap::value_parser!(u64).range(1..))] + pub l1_read_stale_after_blocks: u64, + + /// Assumed L1 block time in seconds. Used to estimate block progression + /// from wall-clock time when the L1 provider is unreachable. + #[arg(long, env = "CARTESI_SEQUENCER_SECONDS_PER_BLOCK", default_value = "12", value_parser = clap::value_parser!(u64).range(1..))] + pub seconds_per_block: u64, +} + +impl TimingArgs { + /// Build a validated [`ProtocolTiming`]. Pure derivation — no I/O. + /// `max_wait_blocks` is the shared scheduler constant; the rest are the + /// operator-tunable CLI args. + pub fn protocol_timing(&self) -> Result { + ProtocolTiming::try_new( + sequencer_core::MAX_WAIT_BLOCKS, + self.preemptive_margin_blocks, + self.l1_read_stale_after_blocks, + self.seconds_per_block, + ) + } +} + +/// Batch-submitter signing-key source, shared by `run` and `flush-mempool` +/// (both sign L1 transactions). `setup` does NOT use this — it takes the +/// submitter address directly and never signs. +#[derive(Debug, Clone, Args)] +#[command(group( + ArgGroup::new("batch_submitter_key_source") + .args(["batch_submitter_private_key", "batch_submitter_private_key_file"]) + .required(true) + .multiple(false) +))] +pub struct KeyArgs { /// Hex-encoded private key for the batch submitter. #[arg( long, @@ -81,6 +121,206 @@ pub struct RunConfig { group = "batch_submitter_key_source" )] batch_submitter_private_key_file: Option, +} + +impl KeyArgs { + /// Resolve the batch submitter private key from either the inline value or a key file. + pub fn resolve(&self) -> Result { + resolve_key_source( + &self.batch_submitter_private_key, + &self.batch_submitter_private_key_file, + ) + .map(|opt| opt.expect("batch submitter private key is required by CLI arg group")) + } +} + +/// Resolve a batch-submitter key from an inline value or a key file (first line). +/// Returns `Ok(None)` when neither source is set — `KeyArgs` makes that +/// impossible via its required arg group, but `setup --recovery` validates the +/// "exactly one source" rule in code (the group can't be conditionally required +/// at the clap level), so it needs the `None` case. +fn resolve_key_source( + inline: &Option, + file: &Option, +) -> Result, std::io::Error> { + if let Some(file) = file { + let contents = std::fs::read_to_string(file)?; + Ok(Some( + contents.lines().next().unwrap_or("").trim().to_string(), + )) + } else { + Ok(inline.clone()) + } +} + +/// Batch-submitter signing-key source for a context where the key is +/// *conditionally* required — `setup --recovery` needs it (to flush the wallet +/// nonce), plain `setup` must not carry it. clap can't express a +/// conditionally-required arg group, so unlike [`KeyArgs`] neither source is +/// required at the clap level; the owner validates presence + exclusivity via +/// [`is_present`](Self::is_present) / [`both_set`](Self::both_set). +#[derive(Debug, Clone, Args)] +pub struct OptionalKeyArgs { + /// Hex-encoded batch-submitter private key. + #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY")] + batch_submitter_private_key: Option, + /// Path to a file whose first line is the batch-submitter private key. + #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE")] + batch_submitter_private_key_file: Option, +} + +impl OptionalKeyArgs { + /// At least one source is set. + fn is_present(&self) -> bool { + self.batch_submitter_private_key.is_some() + || self.batch_submitter_private_key_file.is_some() + } + + /// Both sources are set — the owner rejects this (at most one is allowed). + fn both_set(&self) -> bool { + self.batch_submitter_private_key.is_some() + && self.batch_submitter_private_key_file.is_some() + } + + /// Resolve the key if either source is set; `Ok(None)` when neither is. + fn resolve_if_present(&self) -> Result, std::io::Error> { + resolve_key_source( + &self.batch_submitter_private_key, + &self.batch_submitter_private_key_file, + ) + } +} + +/// `setup` — establish the deployment's timeless state: pin identity, do the +/// initial L1 sync, register the genesis finalized snapshot, write the +/// setup-complete marker. L1-read-only: takes the batch-submitter address, not +/// the signing key. +#[derive(Debug, Clone, Args)] +pub struct SetupConfig { + #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] + pub data_dir: String, + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + pub eth_rpc_url: String, + /// Error codes that trigger `get_logs` retries with a shorter block range. + #[arg(long, env = "CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES", value_delimiter = ',', default_values = crate::l1::partition::DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES)] + pub long_block_range_error_codes: Vec, + /// Expected chain ID. Validated against the RPC at setup and pinned. + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_ID")] + pub chain_id: u64, + /// Application (EIP-712 verifying contract) address. + #[arg(long, env = "CARTESI_SEQUENCER_APP_ADDRESS", value_parser = parse_address)] + pub app_address: Address, + /// Address that submits batches. Pinned into the deployment identity. + /// `setup` never signs, so it takes the address (derivable from the key + /// without the secret) rather than the key itself. + #[arg(long, env = "CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS", value_parser = parse_address)] + pub batch_submitter_address: Address, + /// L1 inclusion block `B` of the trusted checkpoint machine this setup + /// boots from. Default `0` = genesis bootstrap. Used only as the lower + /// bound of `setup`'s read-only detection scan: if a previous instance + /// left any batch-submitter tx past `B` (or its wallet nonce is unsettled), + /// `setup` refuses and points the operator at recovery. PR3 does not yet + /// load a non-genesis checkpoint machine — that is `setup --recovery` (PR5); + /// here `B` only scopes detection, so `B > 0` against a genesis-style setup + /// merely narrows the scan. + #[arg(long, env = "CARTESI_SEQUENCER_CHECKPOINT_BLOCK", default_value_t = 0)] + pub checkpoint_block: u64, + /// Recovery mode (cockroach recovery): rebuild this freshly-wiped DB from a + /// trusted checkpoint at `--checkpoint-block` instead of genesis bootstrap. + /// Requires `--checkpoint-dump-dir`, `--checkpoint-block > 0`, and the + /// batch-submitter signing key (recovery flushes the wallet nonce, which + /// signs L1 no-ops). Plain `setup` stays L1-read-only and key-less. + #[arg(long, env = "CARTESI_SEQUENCER_RECOVERY", default_value_t = false)] + pub recovery: bool, + /// Directory of the trusted checkpoint dump (a finalized `dumps//` with + /// `state` + `info.toml`) that `setup --recovery` boots the machine `S` + /// from. Required with `--recovery`; rejected without it. + #[arg(long, env = "CARTESI_SEQUENCER_CHECKPOINT_DUMP_DIR", value_parser = parse_non_empty_string)] + pub checkpoint_dump_dir: Option, + /// Batch-submitter signing key, recovery-only (for the wallet-nonce flush). + /// Conditionally required: present iff `--recovery`. The two sources are + /// mutually exclusive and rejected on a non-recovery `setup` — see + /// [`SetupConfig::validate`]. + #[command(flatten)] + key: OptionalKeyArgs, + #[command(flatten)] + pub timing: TimingArgs, +} + +impl SetupConfig { + pub fn db_path(&self) -> String { + db_path_in(&self.data_dir) + } + + /// Validate the cross-field recovery constraints clap can't express (a + /// conditionally-required arg group). Returns the first violation as a + /// human-readable message; the caller maps it to a terminal bootstrap error. + /// + /// Recovery requires a real checkpoint (`--checkpoint-block > 0` + a dump + /// dir) and the signing key; a plain `setup` must carry none of the + /// recovery-only inputs (key, dump dir). + pub fn validate(&self) -> Result<(), String> { + let key_present = self.key.is_present(); + if self.key.both_set() { + return Err( + "--batch-submitter-private-key and --batch-submitter-private-key-file \ + are mutually exclusive" + .to_string(), + ); + } + if self.recovery { + if self.checkpoint_dump_dir.is_none() { + return Err("--recovery requires --checkpoint-dump-dir".to_string()); + } + if self.checkpoint_block == 0 { + return Err("--recovery requires --checkpoint-block > 0 \ + (recovery boots a non-genesis checkpoint)" + .to_string()); + } + if !key_present { + return Err("--recovery requires the batch-submitter signing key \ + (--batch-submitter-private-key[-file]) to flush the wallet nonce" + .to_string()); + } + } else { + if self.checkpoint_dump_dir.is_some() { + return Err("--checkpoint-dump-dir is only valid with --recovery".to_string()); + } + if key_present { + return Err("plain `setup` is L1-read-only and takes no signing key; \ + pass the batch-submitter address, or use --recovery" + .to_string()); + } + } + Ok(()) + } + + /// Resolve the recovery signing key. Precondition: [`SetupConfig::validate`] + /// passed with `recovery == true` (so exactly one source is set). + pub fn resolve_recovery_key(&self) -> Result { + self.key + .resolve_if_present() + .map(|opt| opt.expect("recovery key presence is validated before resolve")) + } +} + +/// `run` — boot the sequencer from an already-set-up DB. Identity (chain id, +/// app address, InputBox address, genesis block, submitter address) is read +/// from the DB, so those are NOT CLI args here. Keeps the signing key (run +/// submits) and the runtime tuning knobs. +#[derive(Debug, Clone, Args)] +pub struct RunConfig { + #[arg(long, env = "CARTESI_SEQUENCER_HTTP_ADDR", default_value = DEFAULT_HTTP_ADDR, value_parser = parse_non_empty_string)] + pub http_addr: String, + #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] + pub data_dir: String, + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + pub eth_rpc_url: String, + /// Error codes that trigger `get_logs` retries with a shorter block range. + #[arg(long, env = "CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES", value_delimiter = ',', default_values = crate::l1::partition::DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES)] + pub long_block_range_error_codes: Vec, + #[command(flatten)] + key: KeyArgs, /// How often the batch submitter polls for new work when idle. #[arg( @@ -98,73 +338,69 @@ pub struct RunConfig { )] pub batch_submitter_confirmation_depth: u64, - /// Blocks before MAX_WAIT_BLOCKS to trigger preemptive recovery. - /// The danger threshold is MAX_WAIT_BLOCKS minus this margin. - /// Must be less than MAX_WAIT_BLOCKS (validated at startup). - /// - /// Default 300 (~1h at 12s/block) is sized to give operators meaningful - /// runway to investigate before the system gives up on the current - /// batches — see `docs/recovery/README.md` "Step 1: Danger threshold" - /// for the rationale. + /// Force the inclusion lane to seal the open batch after this many seconds, + /// regardless of size — the liveness bound that caps batch-posting latency + /// for a low-traffic deployment. Default 7200 (2h); operators trade latency + /// against L1 cost, and tests set it small to seal batches promptly. #[arg( long, - env = "CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS", - default_value = "300" + env = "CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS", + default_value = "7200", + value_parser = clap::value_parser!(u64).range(1..) )] - pub preemptive_margin_blocks: u64, - - /// Blocks of safe-head age after which the L1 read view is considered too - /// stale to trust. Independent of the preemptive margin — a separate - /// concern ("how old is the cached L1 view before we stop trusting it" vs. - /// "how much runway before write-side recovery trips"). Must be strictly - /// less than the danger threshold (validated at startup). - /// - /// Default 600 (~2h at 12s/block). - #[arg(long, env = "CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS", default_value = "600", value_parser = clap::value_parser!(u64).range(1..))] - pub l1_read_stale_after_blocks: u64, + pub max_batch_open_seconds: u64, - /// Assumed L1 block time in seconds. Used to estimate block progression from - /// wall-clock time when the L1 provider is unreachable. - #[arg(long, env = "CARTESI_SEQUENCER_SECONDS_PER_BLOCK", default_value = "12", value_parser = clap::value_parser!(u64).range(1..))] - pub seconds_per_block: u64, + #[command(flatten)] + pub timing: TimingArgs, } impl RunConfig { - pub fn build_domain(&self) -> Eip712Domain { - sequencer_core::build_input_domain(self.chain_id, self.app_address) + /// Full path to the SQLite database file inside `data_dir`. + pub fn db_path(&self) -> String { + db_path_in(&self.data_dir) + } + + /// Force-close-an-open-batch deadline as a [`Duration`] (the liveness bound + /// the inclusion lane applies — see [`CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`]). + pub fn max_batch_open(&self) -> std::time::Duration { + std::time::Duration::from_secs(self.max_batch_open_seconds) } /// Build a validated [`ProtocolTiming`] from this config's tuning fields. - /// Pure derivation — does not touch I/O. `max_wait_blocks` is the shared - /// scheduler constant; the rest come from the operator-tunable CLI args. pub fn protocol_timing(&self) -> Result { - ProtocolTiming::try_new( - sequencer_core::MAX_WAIT_BLOCKS, - self.preemptive_margin_blocks, - self.l1_read_stale_after_blocks, - self.seconds_per_block, - ) + self.timing.protocol_timing() } - /// Full path to the SQLite database file inside `data_dir`. + /// Resolve the batch submitter private key from either the inline value or a key file. + pub fn resolve_private_key(&self) -> Result { + self.key.resolve() + } +} + +/// `flush-mempool` — settle the batch-submitter wallet nonce on demand +///. Reads the submitter address + watermark from the DB; signs +/// no-op transactions, so it needs the key. +#[derive(Debug, Clone, Args)] +pub struct FlushConfig { + #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] + pub data_dir: String, + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + pub eth_rpc_url: String, + #[command(flatten)] + key: KeyArgs, + /// Assumed L1 block time in seconds; sets the flusher's confirmation / + /// safe-poll cadence. + #[arg(long, env = "CARTESI_SEQUENCER_SECONDS_PER_BLOCK", default_value = "12", value_parser = clap::value_parser!(u64).range(1..))] + pub seconds_per_block: u64, +} + +impl FlushConfig { pub fn db_path(&self) -> String { - std::path::Path::new(&self.data_dir) - .join(DB_FILENAME) - .to_string_lossy() - .into_owned() + db_path_in(&self.data_dir) } - /// Resolve the batch submitter private key from either the inline value or a key file. pub fn resolve_private_key(&self) -> Result { - if let Some(file) = &self.batch_submitter_private_key_file { - let contents = std::fs::read_to_string(file)?; - Ok(contents.lines().next().unwrap_or("").trim().to_string()) - } else { - Ok(self - .batch_submitter_private_key - .clone() - .expect("batch submitter private key is required by CLI arg group")) - } + self.key.resolve() } } @@ -191,103 +427,231 @@ fn parse_address(raw: &str) -> Result { #[cfg(test)] mod tests { - use super::RunConfig; - use alloy_primitives::{Address, U256}; + use super::*; + use crate::harness::Cli; use clap::Parser; - use sequencer_core::{DOMAIN_NAME, DOMAIN_VERSION}; - const TEST_ARGS: [&str; 9] = [ + // Parse a full subcommand line through the top-level `Cli`, since the + // per-subcommand configs derive `Args` (embeddable) not `Parser`. + const SETUP_ARGS: [&str; 10] = [ "sequencer", + "setup", "--eth-rpc-url", "http://127.0.0.1:8545", "--chain-id", "31337", "--app-address", "0x1111111111111111111111111111111111111111", + "--batch-submitter-address", + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + ]; + + const RUN_ARGS: [&str; 5] = [ + "sequencer", + "run", + "--eth-rpc-url", + "http://127.0.0.1:8545", "--batch-submitter-private-key", - "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", ]; - #[test] - fn run_config_requires_essential_inputs() { - let err = RunConfig::try_parse_from([ - "sequencer", - "--batch-submitter-private-key", - "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - ]) - .expect_err("essential inputs are required"); + const TEST_KEY: &str = "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - let message = err.to_string(); - assert!(message.contains("--eth-rpc-url")); - assert!(message.contains("--chain-id")); - assert!(message.contains("--app-address")); + fn run_config_from(extra: &[&str]) -> RunConfig { + let mut args: Vec<&str> = RUN_ARGS.to_vec(); + args.push(TEST_KEY); + args.extend_from_slice(extra); + match Cli::try_parse_from(args).expect("parse run").command { + crate::harness::Command::Run(c) => *c, + other => panic!("expected run subcommand, got {other:?}"), + } } - #[test] - fn run_config_uses_default_block_range_retry_codes() { - let config = RunConfig::try_parse_from(TEST_ARGS).expect("parse run config"); + fn setup_config() -> SetupConfig { + setup_config_from(&[]) + } + + fn setup_config_from(extra: &[&str]) -> SetupConfig { + let mut args: Vec<&str> = SETUP_ARGS.to_vec(); + args.extend_from_slice(extra); + match Cli::try_parse_from(args).expect("parse setup").command { + crate::harness::Command::Setup(c) => *c, + other => panic!("expected setup subcommand, got {other:?}"), + } + } + #[test] + fn setup_requires_address_not_key() { + let cfg = setup_config(); + assert_eq!(cfg.chain_id, 31337); assert_eq!( - config.long_block_range_error_codes, - vec![ - "-32005".to_string(), - "-32600".to_string(), - "-32602".to_string(), - "-32616".to_string() - ] + cfg.batch_submitter_address, + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + .parse::

() + .unwrap() ); + // The default db path lives under the default data dir. + assert!(cfg.db_path().ends_with("sequencer.db")); + } + + #[test] + fn setup_checkpoint_block_defaults_to_genesis_and_parses() { + // Default = 0 (genesis bootstrap) when the flag is absent. + assert_eq!(setup_config().checkpoint_block, 0); + // Explicit value parses. + let mut args: Vec<&str> = SETUP_ARGS.to_vec(); + args.push("--checkpoint-block"); + args.push("4200"); + let cfg = match Cli::try_parse_from(args).expect("parse setup").command { + crate::harness::Command::Setup(c) => *c, + other => panic!("expected setup subcommand, got {other:?}"), + }; + assert_eq!(cfg.checkpoint_block, 4200); } #[test] - fn run_config_defaults_batch_submitter_confirmation_depth_to_two() { - let config = RunConfig::try_parse_from(TEST_ARGS).expect("parse run config"); + fn plain_setup_passes_validation_and_is_not_recovery() { + let cfg = setup_config(); + assert!(!cfg.recovery); + assert!(cfg.checkpoint_dump_dir.is_none()); + cfg.validate().expect("plain setup is a valid config"); + } - assert_eq!(config.batch_submitter_confirmation_depth, 2); + #[test] + fn setup_rejects_a_signing_key_without_recovery() { + // The key now parses (recovery needs it), but a non-recovery `setup` + // carrying one is rejected by cross-field validation, not at parse time. + let cfg = setup_config_from(&["--batch-submitter-private-key", TEST_KEY]); + let err = cfg.validate().expect_err("plain setup must reject a key"); + assert!( + err.contains("key-less") || err.contains("L1-read-only"), + "got: {err}" + ); } #[test] - fn run_config_builds_domain_with_fixed_name_and_version() { - let config = RunConfig::try_parse_from(TEST_ARGS).expect("parse run config"); + fn setup_rejects_dump_dir_without_recovery() { + let cfg = setup_config_from(&["--checkpoint-dump-dir", "/tmp/ckpt"]); + let err = cfg + .validate() + .expect_err("dump dir without --recovery must fail"); + assert!(err.contains("--checkpoint-dump-dir"), "got: {err}"); + } - let domain = config.build_domain(); - assert_eq!(domain.name.as_deref(), Some(DOMAIN_NAME)); - assert_eq!(domain.version.as_deref(), Some(DOMAIN_VERSION)); - assert_eq!(domain.chain_id, Some(U256::from(31337_u64))); - assert_eq!( - domain.verifying_contract, - Some(Address::from_slice(&[0x11; 20])) + #[test] + fn recovery_requires_dump_dir_block_and_key() { + // --recovery alone (no dump dir / block / key) is invalid; each missing + // input is reported. + let bare = setup_config_from(&["--recovery"]); + assert!( + bare.validate() + .unwrap_err() + .contains("--checkpoint-dump-dir") ); + + let no_block = setup_config_from(&["--recovery", "--checkpoint-dump-dir", "/tmp/ckpt"]); + assert!( + no_block + .validate() + .unwrap_err() + .contains("--checkpoint-block > 0") + ); + + let no_key = setup_config_from(&[ + "--recovery", + "--checkpoint-dump-dir", + "/tmp/ckpt", + "--checkpoint-block", + "1200", + ]); + assert!(no_key.validate().unwrap_err().contains("signing key")); } - // ── H8 regression: CARTESI_SEQUENCER_SECONDS_PER_BLOCK=0 is rejected by clap ── - // - // The H8 hardening added `value_parser = clap::value_parser!(u64).range(1..)` - // on `seconds_per_block` to prevent a divide-by-zero panic in the - // wall-clock fallback (`elapsed_secs / seconds_per_block`). Without the - // value parser, an operator typo would panic the process during the worst - // possible moment — an L1 outage. These tests lock the clap-level guard. + #[test] + fn recovery_full_config_validates_and_resolves_key() { + let cfg = setup_config_from(&[ + "--recovery", + "--checkpoint-dump-dir", + "/tmp/ckpt", + "--checkpoint-block", + "1200", + "--batch-submitter-private-key", + TEST_KEY, + ]); + cfg.validate().expect("full recovery config is valid"); + assert!(cfg.recovery); + assert_eq!(cfg.checkpoint_block, 1200); + assert_eq!(cfg.checkpoint_dump_dir.as_deref(), Some("/tmp/ckpt")); + assert_eq!(cfg.resolve_recovery_key().expect("resolve key"), TEST_KEY); + } - fn args_with_seconds_per_block(value: &str) -> Vec<&str> { - let mut args: Vec<&str> = TEST_ARGS.to_vec(); - args.push("--seconds-per-block"); - args.push(value); - args + #[test] + fn recovery_rejects_both_key_sources() { + let cfg = setup_config_from(&[ + "--recovery", + "--checkpoint-dump-dir", + "/tmp/ckpt", + "--checkpoint-block", + "1200", + "--batch-submitter-private-key", + TEST_KEY, + "--batch-submitter-private-key-file", + "/tmp/key", + ]); + assert!(cfg.validate().unwrap_err().contains("mutually exclusive")); } - fn args_with_l1_read_stale_after_blocks(value: &str) -> Vec<&str> { - let mut args: Vec<&str> = TEST_ARGS.to_vec(); - args.push("--l1-read-stale-after-blocks"); - args.push(value); - args + #[test] + fn run_requires_a_key_and_drops_identity_args() { + // Key required. + let mut args: Vec<&str> = vec!["sequencer", "run", "--eth-rpc-url", "http://x"]; + assert!( + Cli::try_parse_from(args.clone()).is_err(), + "run requires a batch-submitter key" + ); + // chain-id / app-address are no longer run args. + args.push("--batch-submitter-private-key"); + args.push(TEST_KEY); + args.push("--chain-id"); + args.push("31337"); + assert!( + Cli::try_parse_from(args).is_err(), + "run must reject --chain-id (read from DB now)" + ); } #[test] - fn run_config_rejects_seconds_per_block_zero() { - let err = RunConfig::try_parse_from(args_with_seconds_per_block("0")) - .expect_err("seconds_per_block=0 must be rejected"); + fn run_uses_default_block_range_retry_codes() { + let config = run_config_from(&[]); + assert_eq!( + config.long_block_range_error_codes, + vec![ + "-32005".to_string(), + "-32600".to_string(), + "-32602".to_string(), + "-32616".to_string() + ] + ); + } + + #[test] + fn run_defaults_batch_submitter_confirmation_depth_to_two() { + assert_eq!(run_config_from(&[]).batch_submitter_confirmation_depth, 2); + } + + #[test] + fn timing_args_reject_zero_seconds_per_block() { + let err = Cli::try_parse_from([ + "sequencer", + "run", + "--eth-rpc-url", + "http://x", + "--batch-submitter-private-key", + TEST_KEY, + "--seconds-per-block", + "0", + ]) + .expect_err("seconds_per_block=0 must be rejected"); let message = err.to_string(); - // The exact clap wording depends on the version; the specific field is - // what we want to pin. assert!( message.contains("--seconds-per-block") || message.contains("seconds_per_block"), "error must name the offending field, got: {message}" @@ -295,26 +659,28 @@ mod tests { } #[test] - fn run_config_accepts_seconds_per_block_one() { - // One is the minimum allowed (1..). - let config = - RunConfig::try_parse_from(args_with_seconds_per_block("1")).expect("parse succeeds"); - assert_eq!(config.seconds_per_block, 1); + fn run_default_seconds_per_block_is_12() { + assert_eq!(run_config_from(&[]).timing.seconds_per_block, 12); } #[test] - fn run_config_default_seconds_per_block_is_12() { - let config = RunConfig::try_parse_from(TEST_ARGS).expect("parse run config"); - assert_eq!( - config.seconds_per_block, 12, - "default should reflect Ethereum block time" - ); + fn run_default_l1_read_stale_after_blocks_is_600() { + assert_eq!(run_config_from(&[]).timing.l1_read_stale_after_blocks, 600); } #[test] - fn run_config_rejects_l1_read_stale_after_blocks_zero() { - let err = RunConfig::try_parse_from(args_with_l1_read_stale_after_blocks("0")) - .expect_err("l1_read_stale_after_blocks=0 must be rejected"); + fn timing_args_reject_zero_l1_read_stale_after_blocks() { + let err = Cli::try_parse_from([ + "sequencer", + "run", + "--eth-rpc-url", + "http://x", + "--batch-submitter-private-key", + TEST_KEY, + "--l1-read-stale-after-blocks", + "0", + ]) + .expect_err("l1_read_stale_after_blocks=0 must be rejected"); let message = err.to_string(); assert!( message.contains("--l1-read-stale-after-blocks") @@ -324,16 +690,16 @@ mod tests { } #[test] - fn run_config_default_l1_read_stale_after_blocks_is_600() { - // Independent default (NOT derived from margin) — see field doc. - let config = RunConfig::try_parse_from(TEST_ARGS).expect("parse run config"); - assert_eq!(config.l1_read_stale_after_blocks, 600); - } - - #[test] - fn run_config_accepts_l1_read_stale_after_blocks_one() { - let config = RunConfig::try_parse_from(args_with_l1_read_stale_after_blocks("1")) - .expect("parse succeeds"); - assert_eq!(config.l1_read_stale_after_blocks, 1); + fn domain_uses_fixed_name_and_version() { + use alloy_primitives::U256; + use sequencer_core::{DOMAIN_NAME, DOMAIN_VERSION}; + let domain = sequencer_core::build_input_domain(31337, Address::from_slice(&[0x11; 20])); + assert_eq!(domain.name.as_deref(), Some(DOMAIN_NAME)); + assert_eq!(domain.version.as_deref(), Some(DOMAIN_VERSION)); + assert_eq!(domain.chain_id, Some(U256::from(31337_u64))); + assert_eq!( + domain.verifying_contract, + Some(Address::from_slice(&[0x11; 20])) + ); } } diff --git a/sequencer/src/runtime/error.rs b/sequencer/src/runtime/error.rs index 5d9337e..f1de87f 100644 --- a/sequencer/src/runtime/error.rs +++ b/sequencer/src/runtime/error.rs @@ -15,8 +15,8 @@ use thiserror::Error; use crate::ingress::inclusion_lane::InclusionLaneError; use crate::l1::reader::InputReaderError; -use crate::l1::submitter::BatchSubmitterError; -use crate::recovery::{DangerDetectorError, RecoveryError}; +use crate::l1::submitter::{BatchPosterError, BatchSubmitterError}; +use crate::recovery::{DangerDetectorError, RecoveryError, RefuseReason}; use crate::storage::{DangerStatus, DeploymentIdentity, StorageOpenError}; use sequencer_core::protocol::ProtocolTimingError; @@ -42,6 +42,141 @@ pub enum RunError { AppBootstrap(#[from] sequencer_core::application::AppError), } +// ── R4 exit-code projection (WP10) ───────────────────────────────────── +// +// The orchestrator contract: a pure projection of `RunError` into a process +// exit code so the supervisor can tell "restart me" from "restarting is +// futile" without parsing logs. Lives here (one match), called by the harness +// so every app binary inherits it. The exit code is an ops hint, never +// protocol — authority over what the next boot does stays with startup's own +// `check_danger`. Reserved: 1 (unclassified), 2 (clap usage), 101 (panic). + +/// Restart with backoff; a recovery boot is expected next (it may take 15+ +/// min: flush + safe-finality wait). Startup probes must accommodate it. +pub const EXIT_RESTART_EXPECT_RECOVERY: u8 = 10; +/// Restart with backoff; a transient refusal that self-heals when the L1 view +/// freshens. Alert only if it persists. +pub const EXIT_RESTART_TRANSIENT: u8 = 20; +/// Terminal — do not restart; page an operator. The state cannot self-heal. +pub const EXIT_TERMINAL: u8 = 30; +/// Sticky — do **not** auto-restart-loop; an operator must run +/// `setup --recovery`. The recovery sibling of [`EXIT_RESTART_EXPECT_RECOVERY`] +/// (10), but *operator-initiated*: 10 means "restart and `run()` auto-recovers"; +/// 40 means "a previous instance left work past the checkpoint, and only an +/// explicit `setup --recovery` (PR5) can resolve it" — a plain restart of +/// `setup` would re-detect and re-refuse forever. Distinct from terminal (30) +/// in that a known recovery command *does* fix it. +pub const EXIT_SETUP_NEEDS_RECOVERY: u8 = 40; +/// Unclassified failure (worker crash, provider error). Restart with backoff. +pub const EXIT_UNCLASSIFIED: u8 = 1; + +impl RunError { + /// Project this error onto the R4 exit-code contract. Clean shutdown + /// (exit 0) is handled by the caller — a `RunError` is always a failure. + pub fn exit_code(&self) -> u8 { + match self { + RunError::Worker(WorkerExit::DangerDetector(DangerDetectorExit::DangerDetected { + status, + })) => danger_exit_code(status), + // A wrong-chain RPC caught by the reader after boot (the warm-boot + // deferral's backstop) is terminal, like the boot-time + // `BootstrapError::ChainIdMismatch` — a misconfig the operator must + // fix; a blind restart re-hits the same wrong chain. + RunError::Worker(WorkerExit::InputReader(InputReaderExit::Source( + InputReaderError::ChainIdMismatch { .. }, + ))) => EXIT_TERMINAL, + // Same for the submitter's pre-send chain-id gate: a wrong-chain RPC + // is an operator misconfig, terminal rather than a restart loop that + // keeps refusing to sign onto it. + RunError::Worker(WorkerExit::BatchSubmitter(BatchSubmitterExit::Source( + BatchSubmitterError::Poster(BatchPosterError::ChainIdMismatch { .. }), + ))) => EXIT_TERMINAL, + RunError::Bootstrap(b) => bootstrap_exit_code(b), + // Worker crashes, provider errors, IO/storage/app catch-alls. + RunError::Worker(_) + | RunError::Io(_) + | RunError::Storage(_) + | RunError::AppBootstrap(_) => EXIT_UNCLASSIFIED, + } + } +} + +fn danger_exit_code(status: &DangerStatus) -> u8 { + match status { + // A doomed closed batch / aging Tip: the next boot legitimately runs + // a slow recovery (flush + cascade). + DangerStatus::ClosedBatchInDanger(_) | DangerStatus::TipInDanger(_) => { + EXIT_RESTART_EXPECT_RECOVERY + } + // View-dependent refusals that self-heal once the provider recovers. + DangerStatus::L1ViewStale | DangerStatus::EstimatedBatchInDanger(_) => { + EXIT_RESTART_TRANSIENT + } + // The only genuinely terminal danger: canonical divergence (R2). + DangerStatus::CanonicalDivergence(_) => EXIT_TERMINAL, + // `Safe` is never a detector exit. If it ever reaches here the danger + // classification is self-contradicting — page an operator (EXIT_TERMINAL) + // rather than silently restart-loop with backoff (EXIT_UNCLASSIFIED). + DangerStatus::Safe => { + debug_assert!(false, "danger_exit_code called with DangerStatus::Safe"); + EXIT_TERMINAL + } + } +} + +fn bootstrap_exit_code(err: &BootstrapError) -> u8 { + match err { + // Transient: self-heal when the L1 view / provider recovers. + BootstrapError::ChainIdRpc { .. } + | BootstrapError::Identity(IdentityError::FirstBootRequiresL1) + | BootstrapError::DetectionNonceRead { .. } + | BootstrapError::Flush(_) => EXIT_RESTART_TRANSIENT, + BootstrapError::Recovery(RecoveryError::Refuse( + RefuseReason::L1ViewStale | RefuseReason::EstimatedBatchInDanger { .. }, + )) => EXIT_RESTART_TRANSIENT, + // A flush failure or a re-sync lagging the flush view during startup + // recovery is transient — the respawn retries with a fresher view. + // (Same class the `flush-mempool` subcommand gives a `FlushError`.) + BootstrapError::Recovery( + RecoveryError::Flush(_) | RecoveryError::ResyncBehindFlushView { .. }, + ) => EXIT_RESTART_TRANSIENT, + + // Terminal: needs an operator (wrong config, divergence, or a DB that + // was never set up). + BootstrapError::ChainIdMismatch { .. } + | BootstrapError::InvalidProtocolTiming(_) + | BootstrapError::SetupNotComplete + | BootstrapError::CheckpointBeforeGenesis { .. } + | BootstrapError::SetupRecovery(_) + | BootstrapError::Identity(IdentityError::Mismatch { .. } | IdentityError::OrphanedState) => { + EXIT_TERMINAL + } + + // Sticky setup refusal: a previous instance left work past the + // checkpoint. Distinct from the auto-recovery class (10) — a plain + // restart re-refuses; only `setup --recovery` (PR5) resolves it. + BootstrapError::SetupRefuse(_) => EXIT_SETUP_NEEDS_RECOVERY, + BootstrapError::Recovery(RecoveryError::Refuse(RefuseReason::CanonicalDivergence { + .. + })) => EXIT_TERMINAL, + // A wrong-chain RPC caught by the reader *during* a startup-recovery sync + // is terminal, like the boot-time and worker-path ChainIdMismatch arms — + // an operator misconfig a blind restart re-hits (without this it would + // fall to the catch-all below and loop on the wrong chain). + BootstrapError::Recovery(RecoveryError::InputReader( + InputReaderError::ChainIdMismatch { .. }, + )) => EXIT_TERMINAL, + // Same class for the preemptive-recovery flush's pre-signing chain-id + // gate: a mismatch is an operator misconfig, terminal rather than a + // restart loop on the wrong chain. (An RPC *error* during the check + // surfaces as `RecoveryError::Provider` and falls to the retry arm.) + BootstrapError::Recovery(RecoveryError::ChainIdMismatch { .. }) => EXIT_TERMINAL, + + // Other recovery / storage-open failures: unclassified, retry. + BootstrapError::Recovery(_) | BootstrapError::OpenStorage(_) => EXIT_UNCLASSIFIED, + } +} + // ── Bootstrap-phase errors ───────────────────────────────────────────── /// Anything that can go wrong before runtime workers start: config validation, @@ -50,7 +185,7 @@ pub enum RunError { pub enum BootstrapError { #[error(transparent)] OpenStorage(#[from] StorageOpenError), - #[error("RPC chain ID {rpc} does not match --chain-id {config}")] + #[error("RPC chain ID {rpc} does not match the expected chain ID {config}")] ChainIdMismatch { rpc: u64, config: u64 }, /// `eth_chainId` failed on a reachable RPC. We treat this as fatal /// rather than warn-and-continue: proceeding with an unverified chain id @@ -70,6 +205,168 @@ pub enum BootstrapError { /// Deployment-identity guards — see [`IdentityError`]. #[error(transparent)] Identity(#[from] IdentityError), + /// `run` (or `flush-mempool`) was invoked against a DB where `setup` + /// has not completed — the `setup_complete` marker is absent (setup + /// never ran, or crashed midway), or its outputs are incomplete. The + /// operator must run `setup` first; restarting `run` cannot self-heal. + #[error("setup has not completed for this data dir — run `setup` first")] + SetupNotComplete, + /// The `flush-mempool` subcommand's flush failed (provider/transport). + #[error("mempool flush failed: {0}")] + Flush(#[from] crate::recovery::FlushError), + /// `setup`'s detection gate could not read the batch-submitter wallet + /// nonce from L1 (the one live RPC the gate makes). Transient — the + /// operator retries once the provider recovers; the prior sync already + /// proved L1 reachable, so this is a hiccup, not a misconfig. + #[error("setup detection: could not read submitter nonce from RPC: {message}")] + DetectionNonceRead { message: String }, + /// `setup`'s read-only detection gate found a previous instance left work + /// this checkpoint cannot account for. Sticky: only `setup --recovery` + /// (PR5) resolves it — a plain `setup` restart re-detects and re-refuses. + #[error(transparent)] + SetupRefuse(#[from] SetupRefuse), + /// `setup --checkpoint-block` predates the InputBox genesis block: a + /// promotion cannot have landed before the deployment's InputBox existed. + /// Operator misconfig; restarting cannot self-heal. + #[error( + "checkpoint block {checkpoint_block} predates InputBox genesis block \ + {genesis_block}" + )] + CheckpointBeforeGenesis { + checkpoint_block: u64, + genesis_block: u64, + }, + /// `setup --recovery` failed in a way only the operator can fix (bad config, + /// a checkpoint that can't be loaded or doesn't fit the chain, or a DB that + /// is already set up). Terminal — see [`SetupRecoveryError`]. + #[error(transparent)] + SetupRecovery(#[from] SetupRecoveryError), +} + +/// Terminal failures of the `setup --recovery` procedure — the ones +/// an operator must resolve (the flush and the post-flush re-sync reuse the +/// transient [`RecoveryError`] paths instead). All map to [`EXIT_TERMINAL`]: +/// a plain restart re-runs the same bad inputs and re-fails identically. +#[derive(Debug, Error)] +pub enum SetupRecoveryError { + /// Cross-field config validation failed (recovery missing its dump dir / + /// checkpoint block / key, or a plain `setup` carrying recovery-only args). + /// See [`crate::runtime::config::SetupConfig::validate`]. + #[error("invalid recovery configuration: {message}")] + InvalidConfig { message: String }, + /// `setup --recovery` was invoked against a DB that is already set up. + /// Recovery is a strict one-shot on a freshly-wiped DB (wipe and re-run with + /// `--recovery`); re-pointing a live deployment at a different checkpoint + /// would strand its existing state. + #[error( + "`setup --recovery` requires a freshly-wiped data dir, but this one is \ + already set up — wipe it and re-run" + )] + AlreadySetUp, + /// The checkpoint dump could not be loaded (missing/corrupt `info.toml`, or + /// the app's `from_dump` failed). Operator must supply a valid dump dir. + #[error("failed to load checkpoint dump at {path}: {message}")] + CheckpointLoad { path: String, message: String }, + /// The checkpoint's last-executed safe block `A` is not strictly before the + /// checkpoint block `B`. The fold reconstructs the `(A, B]` fridge, so + /// `A < B` must hold — otherwise the checkpoint dump and + /// `--checkpoint-block` describe inconsistent points. + #[error( + "checkpoint last-executed safe block {executed_safe_block} (A) is not \ + before checkpoint block {checkpoint_block} (B)" + )] + CheckpointNotBeforeBlock { + executed_safe_block: u64, + checkpoint_block: u64, + }, + /// A re-run of `setup --recovery` found a root tip from a *prior* (crashed + /// before the `setup_complete` marker) attempt whose nonce differs from this + /// attempt's resume nonce — a different checkpoint, or the same one after the + /// post-flush head `C` advanced. The half-recovered DB cannot be resumed + /// onto a tree rooted at the old nonce (the anchor would move but the + /// existing root tip would not, silently breaking I16). Wipe the data dir + /// and re-run. + #[error( + "partial recovery: existing root tip carries nonce {existing_root_nonce}, \ + but this attempt resumes at {requested_nonce} — wipe the data dir and re-run" + )] + PartialRecoveryMismatch { + existing_root_nonce: u64, + requested_nonce: u64, + }, + /// A re-run of `setup --recovery` found a root tip carrying *this* attempt's + /// resume nonce but **no finalized snapshot** — a prior attempt that crashed + /// between opening the root tip and writing the snapshot. It cannot be + /// resumed safely: a re-sync may have advanced `C` with new direct inputs + /// (which leave `N'` unchanged) that resuming would leave unsequenced, so the + /// snapshot cursor would lag the folded `S'` and `run` would drain+execute + /// them a second time (divergence). Wipe the data dir and re-run (the + /// one-shot recovery model). + #[error( + "partial recovery: root tip at nonce {root_nonce} exists with no finalized \ + snapshot (crashed mid-fill) — wipe the data dir and re-run" + )] + PartialRecoveryIncomplete { root_nonce: u64 }, + /// `setup --recovery` found a finalized snapshot but **no root tip**. A + /// completed cockroach fill always has both (the tip is opened in step 2, + /// before the snapshot in step 4), so this is residue from a *different* + /// deployment mode left in the data dir — a plain `setup` that registered the + /// genesis finalized snapshot and crashed before its `setup_complete` marker. + /// Folding `(S', N')` and then silently keeping the old snapshot would mark + /// setup complete over the genesis state instead of the recovered state. Wipe + /// the data dir and re-run `setup --recovery`. + #[error( + "setup --recovery found a finalized snapshot (block {existing_finalized_block}) \ + with no root tip — residue from an incomplete plain `setup`; wipe the data \ + dir and re-run" + )] + RecoveryOverResidualSnapshot { existing_finalized_block: u64 }, + /// A plain (non-recovery) `setup` found a non-zero batch-tree anchor — + /// residue from a `setup --recovery` that crashed before its marker. Booting + /// a genesis deployment over it would root the tree at the recovery nonce + /// instead of 0. Wipe the data dir, then run plain `setup` or re-run + /// `setup --recovery`. + #[error( + "plain setup found batch-tree anchor {anchor} (≠ 0) — leftover from an \ + incomplete `setup --recovery`; wipe the data dir and re-run" + )] + GenesisOverRecoveryResidue { anchor: u64 }, +} + +/// `setup`'s read-only detection gate: the reasons a +/// fresh `setup` refuses because a *previous* instance left work past the +/// checkpoint. The remedy is `setup --recovery` (PR5), which flushes/folds the +/// outstanding batches; a plain `setup` restart re-detects and re-refuses +/// (hence [`EXIT_SETUP_NEEDS_RECOVERY`], not the auto-recovery class 10). +/// +/// Both variants carry diagnostic fields for the refusal log line, mirroring +/// the danger-detector [`RefuseReason`] precedent. +#[derive(Debug, Error)] +pub enum SetupRefuse { + /// Step 1: the batch-submitter wallet nonce is not settled + /// (`pending > safe`) on the local provider — a previous instance left + /// pending or mined-but-unsafe batch txs. Local-view only (review F1): a + /// zombie tx dropped from this provider's pool but alive elsewhere evades + /// this check; bounded at runtime by the content-identity check. + #[error( + "batch-submitter wallet nonce not settled (pending {pending} > safe \ + {safe}) — a previous instance left in-flight batch txs; run \ + `setup --recovery`" + )] + WalletNonceUnsettled { pending: u64, safe: u64 }, + /// Step 2: a batch-submitter tx exists in `(checkpoint_block, safe]` — a + /// previous instance already wrote batches past this checkpoint, so a + /// genesis-style bootstrap would silently diverge from canonical state. + #[error( + "batch-submitter input found at block {found_block} past checkpoint \ + block {checkpoint_block} (safe_input_index {safe_input_index}) — run \ + `setup --recovery`" + )] + BatchPastCheckpoint { + checkpoint_block: u64, + found_block: u64, + safe_input_index: u64, + }, } /// Deployment-identity failure modes. The sequencer pins itself to a specific @@ -283,3 +580,261 @@ impl From for RunError { RunError::Bootstrap(e.into()) } } + +impl From for RunError { + fn from(e: crate::recovery::FlushError) -> Self { + RunError::Bootstrap(BootstrapError::Flush(e)) + } +} + +impl From for RunError { + fn from(e: SetupRefuse) -> Self { + RunError::Bootstrap(BootstrapError::SetupRefuse(e)) + } +} + +impl From for RunError { + fn from(e: SetupRecoveryError) -> Self { + RunError::Bootstrap(BootstrapError::SetupRecovery(e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::recovery::{FlushError, RecoveryError}; + use crate::storage::DeploymentIdentity; + use sequencer_core::protocol::ProtocolTimingError; + + fn danger(status: DangerStatus) -> RunError { + RunError::Worker(WorkerExit::DangerDetector( + DangerDetectorExit::DangerDetected { status }, + )) + } + + fn dummy_identity() -> DeploymentIdentity { + use alloy_primitives::Address; + DeploymentIdentity { + chain_id: 1, + app_address: Address::repeat_byte(0x11), + input_box_address: Address::repeat_byte(0x22), + input_box_genesis_block: 0, + batch_submitter_address: Address::repeat_byte(0x33), + } + } + + #[test] + fn r4_class_10_expect_recovery_boot() { + assert_eq!( + danger(DangerStatus::ClosedBatchInDanger(0)).exit_code(), + EXIT_RESTART_EXPECT_RECOVERY + ); + assert_eq!( + danger(DangerStatus::TipInDanger(3)).exit_code(), + EXIT_RESTART_EXPECT_RECOVERY + ); + } + + #[test] + fn r4_class_20_transient_refusal() { + assert_eq!( + danger(DangerStatus::L1ViewStale).exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + danger(DangerStatus::EstimatedBatchInDanger(2)).exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Recovery(RecoveryError::Refuse( + RefuseReason::L1ViewStale + ))) + .exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Identity(IdentityError::FirstBootRequiresL1)) + .exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::ChainIdRpc { + message: "x".into() + }) + .exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + RunError::from(FlushError::Provider("x".into())).exit_code(), + EXIT_RESTART_TRANSIENT + ); + // A flush failure surfaced via startup recovery must land in the same + // class as the flush-mempool subcommand's FlushError (review M2). + assert_eq!( + RunError::Bootstrap(BootstrapError::Recovery(RecoveryError::Flush( + FlushError::Provider("x".into()) + ))) + .exit_code(), + EXIT_RESTART_TRANSIENT + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Recovery( + RecoveryError::ResyncBehindFlushView { + resynced_safe_block: 1, + flush_observed_safe_block: 2, + } + )) + .exit_code(), + EXIT_RESTART_TRANSIENT + ); + } + + #[test] + fn r4_class_30_terminal_operator_required() { + assert_eq!( + danger(DangerStatus::CanonicalDivergence(0)).exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Recovery(RecoveryError::Refuse( + RefuseReason::CanonicalDivergence { nonce: 0 } + ))) + .exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::SetupNotComplete).exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::ChainIdMismatch { rpc: 1, config: 2 }).exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::InvalidProtocolTiming( + ProtocolTimingError::MarginNotLessThanMaxWait { + margin: 1200, + max_wait: 1200 + } + )) + .exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Identity(IdentityError::OrphanedState)).exit_code(), + EXIT_TERMINAL + ); + // `setup --recovery` operator-fixable failures are terminal (a restart + // re-runs the same bad inputs). + assert_eq!( + RunError::from(SetupRecoveryError::AlreadySetUp).exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { + fields: "chain_id".into(), + stored: Box::new(dummy_identity()), + expected: Box::new(dummy_identity()), + })) + .exit_code(), + EXIT_TERMINAL + ); + // Partial-recovery residue (B1/B2): operator must wipe — terminal. + assert_eq!( + RunError::from(SetupRecoveryError::PartialRecoveryMismatch { + existing_root_nonce: 3, + requested_nonce: 5, + }) + .exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::from(SetupRecoveryError::GenesisOverRecoveryResidue { anchor: 7 }) + .exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::from(SetupRecoveryError::PartialRecoveryIncomplete { root_nonce: 3 }) + .exit_code(), + EXIT_TERMINAL + ); + assert_eq!( + RunError::from(SetupRecoveryError::RecoveryOverResidualSnapshot { + existing_finalized_block: 0, + }) + .exit_code(), + EXIT_TERMINAL + ); + // Reader-level chain-id mismatch (warm-boot backstop) is terminal, like + // the boot-time BootstrapError::ChainIdMismatch. + assert_eq!( + RunError::Worker(WorkerExit::InputReader(InputReaderExit::Source( + InputReaderError::ChainIdMismatch { + rpc: 1, + expected: 31337, + } + ))) + .exit_code(), + EXIT_TERMINAL + ); + // The same mismatch surfacing during a startup-recovery safe-head sync + // (RecoveryError path) is terminal too — not the unclassified Recovery + // catch-all (which would loop on the wrong chain). + assert_eq!( + RunError::Bootstrap(BootstrapError::Recovery(RecoveryError::InputReader( + InputReaderError::ChainIdMismatch { + rpc: 1, + expected: 31337, + } + ))) + .exit_code(), + EXIT_TERMINAL + ); + } + + #[test] + fn r4_class_40_setup_needs_operator_recovery() { + // Sticky setup refusals: operator must run `setup --recovery`, not a + // plain restart (which would re-detect and re-refuse) — so they get a + // dedicated code, distinct from the auto-recovery class (10). + assert_eq!( + RunError::from(SetupRefuse::WalletNonceUnsettled { + pending: 14, + safe: 13, + }) + .exit_code(), + EXIT_SETUP_NEEDS_RECOVERY + ); + assert_eq!( + RunError::from(SetupRefuse::BatchPastCheckpoint { + checkpoint_block: 100, + found_block: 250, + safe_input_index: 7, + }) + .exit_code(), + EXIT_SETUP_NEEDS_RECOVERY + ); + // A checkpoint predating genesis is operator misconfig — terminal (30), + // not a recovery trigger. + assert_eq!( + RunError::Bootstrap(BootstrapError::CheckpointBeforeGenesis { + checkpoint_block: 5, + genesis_block: 10, + }) + .exit_code(), + EXIT_TERMINAL + ); + } + + #[test] + fn r4_class_1_unclassified() { + assert_eq!( + RunError::Io(std::io::Error::other("boom")).exit_code(), + EXIT_UNCLASSIFIED + ); + assert_eq!( + RunError::Worker(WorkerExit::Server(ServerExit::StoppedUnexpectedly)).exit_code(), + EXIT_UNCLASSIFIED + ); + } +} diff --git a/sequencer/src/runtime/flush.rs b/sequencer/src/runtime/flush.rs new file mode 100644 index 0000000..47c2a65 --- /dev/null +++ b/sequencer/src/runtime/flush.rs @@ -0,0 +1,76 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The `flush-mempool` subcommand: settle the batch-submitter +//! wallet nonce on demand — submit no-ops for every unresolved slot and wait +//! until `pending <= safe` and `safe >= watermark + 1`. +//! +//! Operator tooling (unstick a wedged nonce, prep before a decommission) and +//! the same flush `setup --recovery` will run internally. It is a keyed L1 +//! write, so it needs the signing key; it reads the submitter address and the +//! wallet-nonce watermark from the DB, so it refuses unless `setup` completed. +//! Flush-only — it does not cascade (that stays in preemptive recovery). + +use super::config::FlushConfig; +use super::{BootstrapError, RunError, load_setup_identity}; +use crate::l1::provider::VerifiedSignerProviderError; +use crate::recovery::MempoolFlusher; +use crate::storage; + +pub async fn flush_mempool(config: FlushConfig) -> Result<(), RunError> { + let db_path = config.db_path(); + + // Gate on a completed setup and read the pinned submitter address. + let identity = load_setup_identity(&db_path)?; + + // The signing key must match the pinned submitter — flushing under the + // wrong key would settle the wrong account's nonce. + let key = super::verify_submitter_key(config.resolve_private_key()?, &identity)?; + + // The durable flush anchor (review R1a): every slot we ever broadcast + // must resolve at safe depth, regardless of the local pool's memory. + let watermark = { + let mut storage = storage::Storage::open(&db_path)?; + storage.wallet_nonce_watermark()? + }; + + // Wrong-chain RPC guard (review F6): flush broadcasts keyed L1 txs, so — + // like `setup` and `run` — it must confirm the RPC's chain id matches the + // pinned one before signing, or it would burn submitter nonce slots on the + // wrong chain. `create_verified_signer_provider` folds that check into the + // signer build (the one guarded keyed-write entry point); a mismatch is + // terminal (operator misconfig), an RPC error retryable (flush needs L1 + // reachable anyway). + let provider = crate::l1::provider::create_verified_signer_provider( + &config.eth_rpc_url, + &key, + identity.chain_id, + ) + .await + .map_err(|e| match e { + VerifiedSignerProviderError::ChainIdMismatch { rpc, expected } => { + RunError::Bootstrap(BootstrapError::ChainIdMismatch { + rpc, + config: expected, + }) + } + VerifiedSignerProviderError::ChainIdRpc(message) => { + RunError::Bootstrap(BootstrapError::ChainIdRpc { message }) + } + VerifiedSignerProviderError::Create(msg) => RunError::Io(std::io::Error::other(msg)), + })?; + let safe_block = MempoolFlusher::flush_to_safe( + provider, + identity.batch_submitter_address, + config.seconds_per_block, + db_path, + watermark, + ) + .await?; + tracing::info!( + safe_block, + batch_submitter_address = %identity.batch_submitter_address, + "flush-mempool complete — wallet nonce settled" + ); + Ok(()) +} diff --git a/sequencer/src/runtime/mod.rs b/sequencer/src/runtime/mod.rs index 0c15f56..2eb6f1c 100644 --- a/sequencer/src/runtime/mod.rs +++ b/sequencer/src/runtime/mod.rs @@ -1,9 +1,20 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Process orchestration. Three phases: +//! Process orchestration for the `run` subcommand, plus the shared +//! bootstrap helpers used by all three subcommands. //! -//! 1. **Bootstrap**: parse config, validate identity, build the L1 config. +//! The phase split: [`setup`] establishes the timeless deployment +//! state (identity, initial sync, genesis snapshot, `setup_complete` marker); +//! [`run`] boots workers from an already-set-up DB; [`flush`] settles the +//! wallet nonce. The CLI harness that dispatches them lives in +//! [`crate::harness`]. +//! +//! `run`'s phases: +//! +//! 1. **Gate + identity**: refuse unless `setup` completed; read the pinned +//! deployment identity from the DB (chain id / app address are no longer +//! CLI args — they come from the identity). //! 2. **Preemptive recovery**: run the startup recovery procedure //! ([`crate::recovery::run_preemptive_recovery`]). //! 3. **Workers**: hand off to `workers::Workers` for spawn → select → @@ -14,51 +25,97 @@ pub mod clock; pub mod config; pub mod error; +pub mod flush; +pub mod setup; +mod setup_fill; pub mod shutdown; +#[cfg(test)] +pub(crate) mod test_support; mod workers; use std::time::Duration; -use crate::l1::reader::{InputReader, InputReaderConfig, InputReaderError}; +use crate::l1::reader::{InputReader, InputReaderConfig}; use crate::storage::{self, DeploymentIdentity}; use alloy_primitives::Address; use config::{L1Config, RunConfig}; use sequencer_core::application::Application; -use sequencer_core::protocol::ProtocolTiming; pub use error::{ BatchSubmitterExit, BootstrapError, DangerDetectorExit, IdentityError, InputReaderExit, - LaneExit, RunError, ServerExit, WorkerExit, + LaneExit, RunError, ServerExit, SetupRecoveryError, SetupRefuse, WorkerExit, }; use workers::{Workers, WorkersConfig}; -const INPUT_READER_POLL_INTERVAL: Duration = Duration::from_secs(2); +pub(crate) const INPUT_READER_POLL_INTERVAL: Duration = Duration::from_secs(2); -pub async fn run(app: A, config: RunConfig) -> Result<(), RunError> +/// Boot the sequencer from an already-set-up DB. Generic over the app type +/// (for the lane's `from_dump`, the egress state-file path, and the +/// max-payload bound) but takes no app *value* — `setup` already registered +/// the genesis snapshot, so the lane reloads via `A::from_dump`. +pub async fn run(config: RunConfig) -> Result<(), RunError> where A: Application + Clone + Sync + 'static, { - // ── Bootstrap ──────────────────────────────────────────── + // ── Gate + identity ────────────────────────────────────── std::fs::create_dir_all(&config.data_dir)?; let db_path = config.db_path(); - let key = config.resolve_private_key()?; - let batch_submitter_address = batch_submitter_address_from_private_key(&key)?; - - // One ProtocolTiming shared across the whole process. `try_new` validates - // margin/stale relationships up front — including before the startup log - // below — so a bad config produces a clean typed error instead of - // panicking mid-log. let timing = config.protocol_timing()?; - let (mut input_reader, l1_config) = bootstrap_l1_config( - &config, - db_path.as_str(), + // Refuse to boot unless `setup` completed; the identity it pinned + // supplies chain id / app address / InputBox address / genesis block / + // submitter address — none of which are CLI args on `run`. + let identity = load_setup_identity(&db_path)?; + + // `run` holds the signing key (it submits). The key's address must match + // the pinned submitter address — running with the wrong key against a DB + // pinned to another submitter is a fail-loud identity mismatch. + let key = verify_submitter_key(config.resolve_private_key()?, &identity)?; + + // Validate the RPC chain id against the *pinned* chain id when L1 is + // reachable (guards against a wrong-chain RPC after setup, review F6); + // tolerate an unreachable L1 (warm boot — identity is already pinned). The + // tolerated case is backstopped by the input reader, which re-verifies the + // chain id on its first successful contact (`InputReaderConfig::expected_chain_id`), + // so a provider that reconnects on the wrong chain fails loud before + // ingesting any address-filtered foreign logs. + match validate_rpc_chain_id(&config.eth_rpc_url, identity.chain_id).await { + Ok(()) => {} + Err(RunError::Bootstrap(BootstrapError::ChainIdRpc { message })) => { + tracing::warn!( + error = %message, + "L1 unreachable at boot — continuing from pinned deployment identity" + ); + } + Err(other) => return Err(other), + } + + let l1_config = L1Config { + eth_rpc_url: config.eth_rpc_url.clone(), + input_box_address: identity.input_box_address, + app_address: identity.app_address, + batch_submitter_private_key: key, + batch_submitter_address: identity.batch_submitter_address, + chain_id: identity.chain_id, + }; + + // `run` never re-discovers identity from L1 — it builds the reader from + // the pinned InputBox address + genesis block and syncs incrementally. + let mut input_reader = InputReader::from_parts( + InputReaderConfig { + rpc_url: config.eth_rpc_url.clone(), + app_address: identity.app_address, + poll_interval: INPUT_READER_POLL_INTERVAL, + long_block_range_error_codes: config.long_block_range_error_codes.clone(), + expected_chain_id: identity.chain_id, + }, + identity.input_box_address, + identity.input_box_genesis_block, + db_path.clone(), + identity.batch_submitter_address, timing, - batch_submitter_address, - key, - ) - .await?; + ); tracing::info!( http_addr = %config.http_addr, @@ -66,7 +123,7 @@ where eth_rpc_url = %l1_config.eth_rpc_url, input_box_address = %l1_config.input_box_address, input_reader_genesis_block = input_reader.genesis_block(), - chain_id = config.chain_id, + chain_id = identity.chain_id, app_address = %l1_config.app_address, batch_submitter_address = %l1_config.batch_submitter_address, max_wait_blocks = timing.max_wait_blocks, @@ -75,18 +132,30 @@ where "sequencer startup" ); + // Always-load invariant, checked at the gate (before any recovery write): + // setup registers the genesis finalized snapshot, so a marker-present DB + // with no snapshot is a corrupt/incomplete setup. Fail loud here — ahead + // of preemptive recovery's DB mutations — rather than only at the lane. + { + let mut storage = storage::Storage::open(&db_path)?; + if storage.finalized_dump()?.is_none() { + return Err(BootstrapError::SetupNotComplete.into()); + } + } + // ── Preemptive recovery ────────────────────────────────── // See docs/recovery/ for the full design and TLA+ spec. crate::recovery::run_preemptive_recovery(&db_path, &mut input_reader, &l1_config, &timing) .await?; // ── Workers ────────────────────────────────────────────── - let mut workers = Workers::spawn(WorkersConfig { - app, + let domain = sequencer_core::build_input_domain(identity.chain_id, identity.app_address); + let mut workers = Workers::spawn::(WorkersConfig { run_config: config, l1_config, timing, input_reader, + domain, }) .await?; @@ -94,9 +163,11 @@ where workers.finish(first_exit).await } -// ── Bootstrap helpers ────────────────────────────────────────────────── +// ── Bootstrap helpers (shared by setup / run / flush) ────────────────── -fn batch_submitter_address_from_private_key(private_key: &str) -> Result { +pub(crate) fn batch_submitter_address_from_private_key( + private_key: &str, +) -> Result { use alloy::signers::local::PrivateKeySigner; use std::str::FromStr; @@ -105,92 +176,19 @@ fn batch_submitter_address_from_private_key(private_key: &str) -> Result
Result<(InputReader, L1Config), RunError> { - let input_reader_config = InputReaderConfig { - rpc_url: config.eth_rpc_url.clone(), - app_address: config.app_address, - poll_interval: INPUT_READER_POLL_INTERVAL, - long_block_range_error_codes: config.long_block_range_error_codes.clone(), - }; - - let (input_reader, input_box_address) = match InputReader::new( - db_path.to_owned(), - input_reader_config.clone(), - batch_submitter_address, - timing, - ) - .await - { - Ok(reader) => { - let input_box = reader.input_box_address(); - - // Validate chain ID early — before any DB identity writes. - validate_rpc_chain_id(&config.eth_rpc_url, config.chain_id).await?; - - let expected_identity = DeploymentIdentity { - chain_id: config.chain_id, - app_address: config.app_address, - input_box_address: input_box, - input_box_genesis_block: reader.genesis_block(), - batch_submitter_address, - }; - ensure_deployment_identity(db_path, expected_identity)?; - - (reader, input_box) - } - Err(InputReaderError::Provider(e)) => { - tracing::error!( - error = %e, - "L1 unreachable during bootstrap — checking deployment identity" - ); - let cached = cached_deployment_identity( - db_path, - config.chain_id, - config.app_address, - batch_submitter_address, - )?; - let reader = InputReader::from_parts( - input_reader_config, - cached.input_box_address, - cached.input_box_genesis_block, - db_path.to_owned(), - batch_submitter_address, - timing, - ); - (reader, cached.input_box_address) - } - Err(source) => { - // L1 reachable but `InputReader::new` failed for a non-provider - // reason — wrap as a startup-time worker source error. - return Err(RunError::Worker(WorkerExit::InputReader( - InputReaderExit::Source(source), - ))); - } - }; - - let l1_config = L1Config { - eth_rpc_url: config.eth_rpc_url.clone(), - input_box_address, - app_address: config.app_address, - batch_submitter_private_key, - batch_submitter_address, - }; - Ok((input_reader, l1_config)) +/// Gate `run`/`flush` on a completed `setup` and return the pinned identity. +/// A missing marker — or a marker present but no identity (a corrupt / +/// incomplete setup) — is a terminal `SetupNotComplete`: the operator must +/// (re-)run `setup`, not retry `run`. +pub(crate) fn load_setup_identity(db_path: &str) -> Result { + let storage = storage::Storage::open(db_path)?; + if !storage.is_setup_complete()? { + return Err(BootstrapError::SetupNotComplete.into()); + } + match storage.deployment_identity()? { + Some(identity) => Ok(identity), + None => Err(BootstrapError::SetupNotComplete.into()), + } } /// Verify that the RPC's `eth_chainId` matches the configured chain id. @@ -199,7 +197,10 @@ async fn bootstrap_l1_config( /// unverified chain id into storage would poison subsequent L1-unreachable /// boots and issue soft confirmations against the wrong chain. Caller is /// expected to retry on `ChainIdRpc`. -async fn validate_rpc_chain_id(eth_rpc_url: &str, expected: u64) -> Result<(), RunError> { +pub(crate) async fn validate_rpc_chain_id( + eth_rpc_url: &str, + expected: u64, +) -> Result<(), RunError> { use alloy::providers::Provider; let check_provider = crate::l1::provider::create_provider(eth_rpc_url) .map_err(|e| RunError::Io(std::io::Error::other(e)))?; @@ -217,7 +218,10 @@ async fn validate_rpc_chain_id(eth_rpc_url: &str, expected: u64) -> Result<(), R } } -fn ensure_deployment_identity(db_path: &str, expected: DeploymentIdentity) -> Result<(), RunError> { +pub(crate) fn ensure_deployment_identity( + db_path: &str, + expected: DeploymentIdentity, +) -> Result<(), RunError> { let mut storage = storage::Storage::open(db_path)?; if let Some(stored) = storage.deployment_identity()? { return require_deployment_identity_match(stored, expected); @@ -229,27 +233,6 @@ fn ensure_deployment_identity(db_path: &str, expected: DeploymentIdentity) -> Re require_deployment_identity_match(stored, expected) } -fn cached_deployment_identity( - db_path: &str, - chain_id: u64, - app_address: Address, - batch_submitter_address: Address, -) -> Result { - let storage = storage::Storage::open(db_path)?; - let Some(stored) = storage.deployment_identity()? else { - return Err(IdentityError::FirstBootRequiresL1.into()); - }; - let expected = DeploymentIdentity { - chain_id, - app_address, - input_box_address: stored.input_box_address, - input_box_genesis_block: stored.input_box_genesis_block, - batch_submitter_address, - }; - require_deployment_identity_match(stored, expected)?; - Ok(stored) -} - fn require_deployment_identity_match( stored: DeploymentIdentity, expected: DeploymentIdentity, @@ -266,6 +249,26 @@ fn require_deployment_identity_match( .into()) } +/// Keyed-writer preflight shared by `run` and `flush`: confirm a resolved +/// batch-submitter signing `key` signs for the submitter `setup` pinned in +/// `identity`, returning the key on success. Both subcommands broadcast keyed +/// L1 txs, so signing under the wrong key would consume the wrong wallet's +/// nonce slots — a fail-loud identity mismatch, not a recoverable condition. +pub(crate) fn verify_submitter_key( + key: String, + identity: &DeploymentIdentity, +) -> Result { + let key_address = batch_submitter_address_from_private_key(&key)?; + if key_address != identity.batch_submitter_address { + let expected = DeploymentIdentity { + batch_submitter_address: key_address, + ..*identity + }; + require_deployment_identity_match(*identity, expected)?; + } + Ok(key) +} + fn deployment_identity_mismatch_fields( stored: DeploymentIdentity, expected: DeploymentIdentity, diff --git a/sequencer/src/runtime/setup.rs b/sequencer/src/runtime/setup.rs new file mode 100644 index 0000000..0b2fd2f --- /dev/null +++ b/sequencer/src/runtime/setup.rs @@ -0,0 +1,810 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The `setup` subcommand: establish everything timeless +//! and pin it in the DB, then mark setup complete so `run` will boot. +//! +//! Steps, in order — each individually idempotent so a crashed `setup` +//! re-runs cleanly, and the marker written *last* is the single +//! linearization point for "A finished": +//! +//! 1. Validate protocol timing; create the data dir + `dumps/`. +//! 2. Require L1: discover the InputBox address + genesis block from the app +//! contract, validate the RPC chain id. (No cached-identity fallback — +//! this is first-boot; an unreachable L1 is a retryable refusal.) +//! 3. Pin the deployment identity (chain id, app address, InputBox address, +//! genesis block, batch-submitter **address** — `setup` never signs). +//! 4. Initial L1 sync: read all direct inputs up to the current safe head. +//! 5. Register the genesis application state as the finalized snapshot. +//! 6. Write the `setup_complete` marker. +//! +//! `setup` is L1-read-only: it takes the batch-submitter address (not the +//! key) and does no L1 writes. + +use alloy_primitives::Address; +use sequencer_core::application::Application; +use sequencer_core::scheduler::{FoldInput, SchedulerConfig, fold_replay}; + +use super::config::SetupConfig; +use super::{ + BootstrapError, IdentityError, InputReaderExit, RunError, SetupRecoveryError, SetupRefuse, + WorkerExit, ensure_deployment_identity, setup_fill, validate_rpc_chain_id, +}; +use crate::ingress::inclusion_lane::dump_info; +use crate::l1::provider::VerifiedSignerProviderError; +use crate::l1::reader::{InputReader, InputReaderConfig, InputReaderError}; +use crate::recovery::{MempoolFlusher, assert_resync_caught_up}; +use crate::storage::{self, DeploymentIdentity}; + +pub async fn setup(config: SetupConfig, genesis_app: A) -> Result<(), RunError> +where + A: Application + 'static, +{ + // Cross-field config validation (recovery vs the recovery-only args). A + // misconfig is operator error — terminal, before any filesystem touch. + config + .validate() + .map_err(|message| SetupRecoveryError::InvalidConfig { message })?; + + std::fs::create_dir_all(&config.data_dir)?; + let dumps_dir = std::path::Path::new(&config.data_dir).join("dumps"); + std::fs::create_dir_all(&dumps_dir)?; + let db_path = config.db_path(); + let timing = config.timing.protocol_timing()?; + + // The `setup_complete` marker gates re-invocation differently per mode: + // * plain `setup` is idempotent — a re-run on a complete DB is a no-op + // success (a half-finished setup has no marker and re-runs below); + // * `setup --recovery` is a strict one-shot on a freshly-wiped DB — a + // complete DB means recovery already ran (or this is a live deployment), + // and re-pointing it at a different checkpoint would strand its state, + // so refuse (terminal). A crash-*before*-marker recovery has no marker; + // its fill is not blindly idempotent either — the partial/residue cases + // are handled fail-loud by `setup_fill::fill_recovery_state` (see there). + { + let storage = storage::Storage::open(&db_path)?; + if storage.is_setup_complete()? { + if config.recovery { + return Err(SetupRecoveryError::AlreadySetUp.into()); + } + tracing::info!(data_dir = %config.data_dir, "setup already complete — nothing to do"); + return Ok(()); + } + } + + // ── L1 discovery (required) ────────────────────────────── + let input_reader_config = InputReaderConfig { + rpc_url: config.eth_rpc_url.clone(), + app_address: config.app_address, + poll_interval: super::INPUT_READER_POLL_INTERVAL, + long_block_range_error_codes: config.long_block_range_error_codes.clone(), + // `validate_rpc_chain_id` below re-checks this against the live RPC; the + // reader's own check then verifies every sync provider serves it too. + expected_chain_id: config.chain_id, + }; + let mut input_reader = match InputReader::new( + db_path.clone(), + input_reader_config, + config.batch_submitter_address, + timing, + ) + .await + { + Ok(reader) => reader, + Err(InputReaderError::Provider(e)) => { + // First boot needs L1 to discover + pin identity. Retryable: the + // operator brings L1 up and re-runs `setup`. + tracing::error!(error = %e, "L1 unreachable during setup — cannot discover identity"); + return Err(IdentityError::FirstBootRequiresL1.into()); + } + Err(source) => { + return Err(RunError::Worker(WorkerExit::InputReader( + InputReaderExit::Source(source), + ))); + } + }; + + validate_rpc_chain_id(&config.eth_rpc_url, config.chain_id).await?; + + // ── Pin identity ───────────────────────────────────────── + // INVARIANT: identity is pinned (this step) BEFORE the initial sync + // (next step). The sync is the first writer of `safe_inputs` / + // `l1_safe_head`, which `has_persisted_deployment_state` keys on. Pinning + // first means a crash mid-sync re-runs cleanly; reordering would make a + // crashed setup look like `OrphanedState` (DB has state, no identity) on + // the retry. See `ensure_deployment_identity` + docs/invariants.md. + let identity = DeploymentIdentity { + chain_id: config.chain_id, + app_address: config.app_address, + input_box_address: input_reader.input_box_address(), + input_box_genesis_block: input_reader.genesis_block(), + batch_submitter_address: config.batch_submitter_address, + }; + ensure_deployment_identity(&db_path, identity)?; + + // ── Detection gate, step 0: checkpoint sanity ──────────── + // A checkpoint promotion cannot predate the InputBox's own genesis block. + // `B = 0` is the genesis bootstrap (no checkpoint) and is always valid. + // (PR3 detects only; loading a non-genesis checkpoint machine and the + // `A < B` check are `setup --recovery` / PR5.) + if config.checkpoint_block != 0 && config.checkpoint_block < input_reader.genesis_block() { + return Err(BootstrapError::CheckpointBeforeGenesis { + checkpoint_block: config.checkpoint_block, + genesis_block: input_reader.genesis_block(), + } + .into()); + } + + // ── Initial L1 sync ────────────────────────────────────── + // One pass reads every direct input up to the current safe head into + // `safe_inputs` + `safe_accepted_batches` and persists `l1_safe_head`. + // Idempotent: a retried setup resumes from the persisted safe head. A + // transient sync failure leaves the marker unwritten, so a re-run resyncs + // cleanly. + // + // Recovery rebuilds the batch tree from the checkpoint *after* this sync, so + // the local tree is empty here. Disable frontier population for recovery's + // syncs (this one and the post-flush re-sync): otherwise the + // content-identity check would see every L1 batch as "foreign" and falsely + // freeze the frontier, poisoning the rebuilt DB. `run`'s first sync + // populates it correctly once the anchor = N' is set (I16). + if config.recovery { + input_reader.set_frontier_mode(storage::FrontierMode::DeferUntilAnchorSet); + } + input_reader + .sync_to_current_safe_head() + .await + .map_err(|e| RunError::Worker(WorkerExit::InputReader(InputReaderExit::Source(e))))?; + + // ── Branch: recovery rebuild vs the genesis-style detect-and-refuse ── + let mut storage = storage::Storage::open(&db_path)?; + + if config.recovery { + // ── Recovery: flush → fold → fill. Replaces the + // detection gate + genesis snapshot; the operator is acting *on* the + // refusal the gate would otherwise raise. + recover::( + &config, + &identity, + timing, + &mut input_reader, + &mut storage, + &dumps_dir, + ) + .await?; + } else { + // ── Detection gate, steps 1–2: refuse if a previous instance left work ── + // Read-only: no key, no L1 write. Runs *after* + // the sync so step 2 reads a `safe_inputs` table populated to the safe + // head, and *before* the genesis snapshot / marker so a refusing setup + // leaves no marker — a re-run (while still incomplete) re-detects + // identically. (Once the marker *is* written, the idempotent early-return + // above skips the gate; that is correct — the deployment is this + // instance's own, and detecting a dirty chain is the job of a *fresh* + // `setup` whose marker is absent.) + // + // Coherence: read the submitter's `safe` nonce at the **persisted** safe + // block from the sync, not the live `Safe` tag. The head can advance + // between the sync and this read; pinning step 1 to the same block step 2 + // scans (`safe_inputs` up to that block) closes the window where a batch + // landing in between would be counted as settled by a live-tag read yet + // be missing from the not-yet-resynced scan. `pending` stays live, so any + // submitter activity past the synced head still trips `pending > safe`. + // + // F1: step 1 reads the LOCAL provider's pool view — a zombie tx dropped + // from this pool but alive elsewhere evades it, bounded at runtime by the + // content-identity check (CanonicalDivergence → cockroach recovery). + let synced_safe_block = storage.current_safe_block()?; + let nonce_views = read_submitter_nonce_views( + &config.eth_rpc_url, + config.batch_submitter_address, + synced_safe_block, + ) + .await?; + run_detection_gate( + &mut storage, + config.batch_submitter_address, + config.checkpoint_block, + nonce_views, + )?; + + // Refuse to register genesis over leftover recovery state. A + // `setup --recovery` that crashed before its marker leaves a non-zero + // batch-tree anchor (and maybe a root tip); booting genesis-style over + // it would root the tree at the recovery nonce instead of 0. Fail loud + // (the marker is the only "this DB is mine" signal, and it's absent in + // both the fresh-genesis and crashed-recovery cases — so we check the + // anchor explicitly). Operator wipes the data dir and re-runs. + let anchor = storage.batch_tree_anchor()?; + if anchor != 0 { + return Err(SetupRecoveryError::GenesisOverRecoveryResidue { anchor }.into()); + } + + // ── Genesis snapshot ───────────────────────────────────── + setup_fill::register_genesis_finalized_snapshot::( + genesis_app, + &mut storage, + &dumps_dir, + )?; + } + + // ── Marker (the single linearization point for "setup finished") ── + storage.mark_setup_complete()?; + + tracing::info!( + data_dir = %config.data_dir, + chain_id = identity.chain_id, + app_address = %identity.app_address, + input_box_address = %identity.input_box_address, + input_box_genesis_block = identity.input_box_genesis_block, + batch_submitter_address = %identity.batch_submitter_address, + "setup complete" + ); + Ok(()) +} + +/// The trusted checkpoint a `setup --recovery` folds from — the machine state +/// `S` at block `B`, plus the two scalars the fold needs that the bare-metal app +/// cannot recompute: `A` (`S`'s last-executed safe block — the fridge is the +/// `(A, B]` directs) and `N` (the resume batch nonce at `B`, which rides in the +/// dump's `info.toml`). See the data dictionary in `docs/recovery/cockroach.md`. +struct Checkpoint { + /// `S` — the checkpoint app state at block `B`. + app: A, + /// `A` — `S`'s last-executed safe block. + executed_safe_block: u64, + /// `N` — the resume batch nonce at `B` (checkpoint metadata). Named to + /// contrast with the fold's output `N'` (`resume_nonce` in `recover`). + checkpoint_nonce: u64, + /// `B` — the checkpoint's L1 inclusion block (`--checkpoint-block`). + checkpoint_block: u64, +} + +impl Checkpoint { + /// Load `S` from the dump dir, derive `A` and `N`, and enforce the load-time + /// precondition `A < B` (else the `(A, B]` fridge range is ill-defined). All + /// failures are terminal — the operator must supply a valid checkpoint. + fn load(dir: &std::path::Path, checkpoint_block: u64) -> Result { + let load_err = |message: String| SetupRecoveryError::CheckpointLoad { + path: dir.display().to_string(), + message, + }; + let info = dump_info::read_info(dir).map_err(|e| load_err(e.to_string()))?; + let app = A::from_dump(&dump_info::app_prefix(dir)).map_err(|e| load_err(e.to_string()))?; + let executed_safe_block = app.last_executed_safe_block(); + if executed_safe_block >= checkpoint_block { + return Err(SetupRecoveryError::CheckpointNotBeforeBlock { + executed_safe_block, + checkpoint_block, + }); + } + Ok(Self { + app, + executed_safe_block, + checkpoint_nonce: info.next_batch_nonce, + checkpoint_block, + }) + } +} + +/// Source the fold inputs from the synced `safe_inputs`: the +/// `(A, B]` direct **seeds** and the full `(B, C]` **replay** stream. +/// +/// Seeds must be directs only — the fold enqueues them straight into the fridge +/// (`enqueue_direct`, no classification). The `sender != submitter` filter IS +/// the scheduler's own sender-based classification: the fold's `process_input` +/// routes `sender == sequencer_address` to the batch path and everything else to +/// a direct, so a dropped `sender == submitter` row is never a direct-with- +/// effect — it is a batch already folded into `S` (every valid batch with +/// inclusion `≤ B` is in `S` by the operator's checkpoint, the same trust +/// boundary as the resume nonce `N`) or an undecodable scheduler no-op. +/// +/// Replay is the full stream; the fold classifies each input itself, so no +/// pre-filter. +fn source_fold_inputs( + storage: &mut storage::Storage, + checkpoint: &Checkpoint, + stop_block: u64, + submitter: Address, +) -> Result<(Vec, Vec), RunError> { + let seeds = storage + .safe_inputs_in_block_range(checkpoint.executed_safe_block, checkpoint.checkpoint_block)? + .into_iter() + .filter(|input| input.sender != submitter) + .map(to_fold_input) + .collect(); + let replay = storage + .safe_inputs_in_block_range(checkpoint.checkpoint_block, stop_block)? + .into_iter() + .map(to_fold_input) + .collect(); + Ok((seeds, replay)) +} + +/// The `setup --recovery` procedure: rebuild a freshly-wiped DB from +/// a trusted checkpoint instead of refusing. Runs after the shared prefix +/// (identity pinned, initial sync done); replaces the detection gate + genesis +/// snapshot. Distinct, terminal error type ([`SetupRecoveryError`]) from +/// `run`'s recovery — operator-driven, one-shot. +/// +/// The `flush → fold → fill` steps are enumerated authoritatively in +/// **[`docs/recovery/cockroach.md`](../../../docs/recovery/cockroach.md)** (spec, +/// data dictionary `A`/`B`/`C`/`N`/`N'`, and code map) and anchored inline below +/// (`// 1.`…`// 6.`). Read the doc before editing this function. +/// +/// `N` (and the whole checkpoint tuple `S`/`A`/`B`/`N`) is **trusted** metadata +/// from the sequencer-produced finalized dump; recovery does not independently +/// re-verify it (there is no local oracle without replaying the scheduler from +/// genesis). See cockroach.md "Data dictionary" for the trust boundary. +async fn recover( + config: &SetupConfig, + identity: &DeploymentIdentity, + timing: sequencer_core::protocol::ProtocolTiming, + input_reader: &mut InputReader, + storage: &mut storage::Storage, + dumps_dir: &std::path::Path, +) -> Result<(), RunError> +where + A: Application + 'static, +{ + // 1. Load the trusted checkpoint (S, A, N, B); require A < B. + let checkpoint_dir = std::path::Path::new( + config + .checkpoint_dump_dir + .as_deref() + .expect("validated: recovery requires a checkpoint dump dir"), + ); + let checkpoint = Checkpoint::::load(checkpoint_dir, config.checkpoint_block)?; + + // 2. Flush the previous instance's batch txs → C, the post-flush safe head. + let stop_block = + flush_wallet_nonce(config, identity, timing.seconds_per_block, storage).await?; + + // 3. Re-sync through C; F2 coherence. Frontier population stays OFF (the + // caller disabled it before the initial sync): the tree is rebuilt in + // step 6, so a frontier built here against an empty tree would falsely + // diverge. `run`'s first sync populates it correctly once anchor = N'. + tracing::info!("re-syncing L1 safe head after flush"); + input_reader + .sync_to_current_safe_head() + .await + .map_err(|e| RunError::Worker(WorkerExit::InputReader(InputReaderExit::Source(e))))?; + let resynced_safe_block = storage.current_safe_block()?.unwrap_or(0); + assert_resync_caught_up(resynced_safe_block, stop_block)?; + + // 4. Source the (A, B] direct seeds + the (B, C] replay stream. + let submitter = identity.batch_submitter_address; + let (seeds, replay) = source_fold_inputs(storage, &checkpoint, stop_block, submitter)?; + + // 5. Fold (S, N) over the inputs → (S', N'). + let Checkpoint { + app, + executed_safe_block, + checkpoint_nonce, + checkpoint_block, + } = checkpoint; + let domain = sequencer_core::build_input_domain(identity.chain_id, identity.app_address); + let scheduler_config = SchedulerConfig::new(submitter); + let (recovered_app, resume_nonce) = fold_replay( + app, + checkpoint_nonce, + scheduler_config, + domain, + seeds, + replay, + stop_block, + ); + + // 6. Fill the DB: finalized S', tree anchored at N', cursor past the ≤C + // directs (already in S'). run boots from this state, and its first sync + // populates the gold frontier from L1 with the anchor = N' (so the folded + // `< N'` batches are skipped as trusted collapsed history, not foreign). + setup_fill::fill_recovery_state(recovered_app, resume_nonce, stop_block, storage, dumps_dir)?; + + tracing::info!( + executed_safe_block, + checkpoint_block, + stop_block, + resume_nonce, + "recovery complete — DB rebuilt from checkpoint" + ); + Ok(()) +} + +/// Recovery step 2: flush the previous instance's stranded batch txs and wait +/// for the wallet nonce to settle, returning `C` — the post-flush safe head at +/// which every prior batch is resolved at safe depth. This is `setup`'s only +/// L1-signing action (it otherwise takes the submitter *address*, not the key). +/// +/// The flush-acquire core (flusher + watermark sink + `flush_and_wait`) is +/// shared with the runtime danger path and `flush-mempool` via +/// [`MempoolFlusher::flush_to_safe`]; this site supplies the recovery key, the +/// pinned submitter address, and the watermark from the open `storage`. +async fn flush_wallet_nonce( + config: &SetupConfig, + identity: &DeploymentIdentity, + seconds_per_block: u64, + storage: &mut storage::Storage, +) -> Result { + // The recovery key must match the pinned batch-submitter address — flushing + // under a different account's key would settle the wrong wallet's nonce (and + // burn its gas) while recovery never settles `identity.batch_submitter_address`. + // Gate before signing, exactly as `flush-mempool` does. + let key = super::verify_submitter_key(config.resolve_recovery_key()?, identity)?; + // Keyed-write chain-id gate: `setup --recovery`'s flush signs L1 no-op txs, + // so re-confirm the RPC serves the pinned chain right before building the + // signer (the earlier `validate_rpc_chain_id` may have gone stale). Same + // guarded constructor the runtime flush and `flush-mempool` use. + let provider = crate::l1::provider::create_verified_signer_provider( + &config.eth_rpc_url, + &key, + config.chain_id, + ) + .await + .map_err(|e| match e { + VerifiedSignerProviderError::ChainIdMismatch { rpc, expected } => { + RunError::Bootstrap(BootstrapError::ChainIdMismatch { + rpc, + config: expected, + }) + } + VerifiedSignerProviderError::ChainIdRpc(message) => { + RunError::Bootstrap(BootstrapError::ChainIdRpc { message }) + } + VerifiedSignerProviderError::Create(msg) => RunError::Io(std::io::Error::other(msg)), + })?; + let watermark = storage.wallet_nonce_watermark()?; + let stop_block = MempoolFlusher::flush_to_safe( + provider, + identity.batch_submitter_address, + seconds_per_block, + config.db_path(), + watermark, + ) + .await?; + Ok(stop_block) +} + +/// Map a synced `safe_inputs` row to a fold input. The EIP-712 domain is a +/// deployment-wide constant the engine clones onto each, so it is not stored +/// per row. +fn to_fold_input(input: crate::storage::StoredSafeInput) -> FoldInput { + FoldInput { + sender: input.sender, + inclusion_block: input.block_number, + payload: input.payload, + } +} + +/// The read-only detection gate, separated +/// from the L1/RPC I/O so its logic is unit-testable against a seeded DB. +/// +/// `nonce_views` is the submitter's `(pending, safe)` wallet nonce read from +/// L1; `storage` holds the safe inputs synced to the safe head. The two steps +/// are ordered on purpose: step 1 (nonce settled?) gates step 2 (scan). An +/// unsettled nonce refuses *without* scanning — unsafe batches are not yet in +/// `safe_inputs`, so the scan would be incomplete and could false-negative. +fn run_detection_gate( + storage: &mut storage::Storage, + batch_submitter: Address, + checkpoint_block: u64, + (pending_nonce, safe_nonce): (u64, u64), +) -> Result<(), RunError> { + // Step 1 — `pending > safe` means a previous instance left in-flight + // (pending or mined-but-unsafe) batch txs. + if pending_nonce > safe_nonce { + return Err(SetupRefuse::WalletNonceUnsettled { + pending: pending_nonce, + safe: safe_nonce, + } + .into()); + } + // Step 2 — settled ⟹ nothing of ours sits unsafe, so the synced + // `safe_inputs` already reflect every previous batch (scanning them is + // equivalent to scanning `(B, safe]`). Refuse if any batch-submitter tx + // landed strictly past the checkpoint block. The query reads the + // reader-synced table, inheriting the reader's range-completeness hardening + // (F5) — see `first_batch_submitter_input_after_block`. + if let Some((safe_input_index, found_block)) = + storage.first_batch_submitter_input_after_block(batch_submitter, checkpoint_block)? + { + return Err(SetupRefuse::BatchPastCheckpoint { + checkpoint_block, + found_block, + safe_input_index, + } + .into()); + } + Ok(()) +} + +/// Read the batch-submitter wallet nonce at the pending tag and at `safe_block` +/// (read-only): returns `(pending, safe)`. `pending > safe` means a previous +/// instance left batch txs past the synced safe head — the step-1 signal. +/// +/// `safe_block` is the safe head the preceding sync persisted; the `safe` nonce +/// is read at exactly that block (`Number(safe_block)`) so step 1 is coherent +/// with step 2's scan of `safe_inputs` synced to the same block — not the live +/// `Safe` tag, which can advance between the sync and this read and reintroduce +/// a TOCTOU gap. `None` (no safe head persisted — not expected after a +/// successful sync) falls back to the `Safe` tag. +/// +/// Builds a provider from the RPC URL via the same `create_provider` the input +/// reader uses (the reader holds no long-lived provider — it builds one per +/// pass), so no second persistent connection is introduced. Both reads are the +/// batch-submitter's `eth_getTransactionCount`; an RPC failure here is +/// transient ([`BootstrapError::DetectionNonceRead`]) — the prior sync already +/// proved L1 reachable. +async fn read_submitter_nonce_views( + eth_rpc_url: &str, + batch_submitter: Address, + safe_block: Option, +) -> Result<(u64, u64), BootstrapError> { + use alloy::providers::Provider; + use alloy::rpc::types::BlockNumberOrTag; + + let provider = crate::l1::provider::create_provider(eth_rpc_url) + .map_err(|message| BootstrapError::DetectionNonceRead { message })?; + let pending = provider + .get_transaction_count(batch_submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .map_err(|e| BootstrapError::DetectionNonceRead { + message: e.to_string(), + })?; + let safe_tag = match safe_block { + Some(block) => BlockNumberOrTag::Number(block), + None => BlockNumberOrTag::Safe, + }; + let safe = provider + .get_transaction_count(batch_submitter) + .block_id(safe_tag.into()) + .await + .map_err(|e| BootstrapError::DetectionNonceRead { + message: e.to_string(), + })?; + Ok((pending, safe)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::Storage; + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::{SENDER_A, default_protocol_timing, temp_db}; + + /// Seed `safe_inputs` with one batch-submitter (`SENDER_A`) input per block + /// in `blocks`, synced up to the max block. Payloads are junk (scheduler + /// no-ops) — detection scans raw rows, not the accepted frontier. + fn seed_submitter_inputs(storage: &mut Storage, blocks: &[u64]) { + let protocol = default_protocol_timing(); + let inputs: Vec = blocks + .iter() + .map(|&block_number| StoredSafeInput { + sender: SENDER_A, + payload: vec![0x01], + block_number, + }) + .collect(); + let safe_block = blocks.iter().copied().max().unwrap_or(0); + storage + .append_safe_inputs(safe_block, inputs.as_slice(), SENDER_A, &protocol) + .expect("seed safe inputs"); + } + + #[test] + fn detection_passes_when_settled_and_no_batch_past_checkpoint() { + let db = temp_db("detect-pass"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + // Settled (pending == safe) and no submitter inputs at all. + assert!(run_detection_gate(&mut storage, SENDER_A, 0, (5, 5)).is_ok()); + } + + #[test] + fn detection_refuses_unsettled_nonce_before_scanning() { + let db = temp_db("detect-unsettled"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + // Step 1 gates step 2: `pending > safe` refuses regardless of inputs + // (and the DB is empty here, proving step 2 never ran). + match run_detection_gate(&mut storage, SENDER_A, 0, (14, 13)) { + Err(RunError::Bootstrap(BootstrapError::SetupRefuse( + SetupRefuse::WalletNonceUnsettled { pending, safe }, + ))) => assert_eq!((pending, safe), (14, 13)), + other => panic!("expected WalletNonceUnsettled, got {other:?}"), + } + } + + #[test] + fn detection_refuses_batch_past_checkpoint_when_settled() { + let db = temp_db("detect-batch-past"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + seed_submitter_inputs(&mut storage, &[18]); + match run_detection_gate(&mut storage, SENDER_A, 0, (1, 1)) { + Err(RunError::Bootstrap(BootstrapError::SetupRefuse( + SetupRefuse::BatchPastCheckpoint { + checkpoint_block, + found_block, + .. + }, + ))) => { + assert_eq!(checkpoint_block, 0); + assert_eq!(found_block, 18); + } + other => panic!("expected BatchPastCheckpoint, got {other:?}"), + } + } + + #[test] + fn detection_passes_when_batch_at_or_below_checkpoint() { + let db = temp_db("detect-batch-below"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + seed_submitter_inputs(&mut storage, &[18]); + // A batch at block 18 is part of `S` for a checkpoint at 18 (not + // strictly past it), so a settled deployment proceeds. + assert!(run_detection_gate(&mut storage, SENDER_A, 18, (1, 1)).is_ok()); + } + + /// `setup --recovery`'s flush signs L1 no-ops with the recovery key, so it + /// must match the pinned batch-submitter — otherwise it would settle some + /// other account's nonce while recovery never settles the intended wallet. + /// The guard is pure and runs first, so a wrong key is refused before any L1 + /// contact (the RPC here is unreachable) or DB write. + #[tokio::test] + async fn flush_wallet_nonce_refuses_mismatched_recovery_key() { + use crate::harness::{Cli, Command}; + use crate::storage::DeploymentIdentity; + use clap::Parser; + + // A recovery key whose address is NOT the pinned submitter below. + const OTHER_KEY: &str = + "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let pinned_submitter: Address = "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + .parse() + .unwrap(); + + let cli = Cli::try_parse_from([ + "sequencer", + "setup", + "--eth-rpc-url", + "http://127.0.0.1:1", // unreachable — the verify fails before any contact + "--chain-id", + "31337", + "--app-address", + "0x1111111111111111111111111111111111111111", + "--batch-submitter-address", + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "--recovery", + "--checkpoint-block", + "10", + "--checkpoint-dump-dir", + "/nonexistent/checkpoint", // never read — refusal precedes the load + "--batch-submitter-private-key", + OTHER_KEY, + ]) + .expect("parse setup --recovery"); + let config = match cli.command { + Command::Setup(c) => *c, + other => panic!("expected setup subcommand, got {other:?}"), + }; + + let identity = DeploymentIdentity { + chain_id: 31337, + app_address: "0x1111111111111111111111111111111111111111" + .parse() + .unwrap(), + input_box_address: "0x2222222222222222222222222222222222222222" + .parse() + .unwrap(), + input_box_genesis_block: 0, + batch_submitter_address: pinned_submitter, + }; + let db = temp_db("recovery-wrong-key"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + + let result = flush_wallet_nonce(&config, &identity, 12, &mut storage).await; + match result { + Err(RunError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { + fields, + .. + }))) => assert_eq!(fields, "batch_submitter_address"), + other => panic!("expected batch_submitter_address mismatch, got: {other:?}"), + } + } + + #[test] + fn source_fold_inputs_splits_seeds_and_replay_dropping_submitter_batches() { + use crate::storage::test_helpers::SENDER_B; + let db = temp_db("source-fold-inputs"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let protocol = default_protocol_timing(); + let submitter = SENDER_A; + let direct = SENDER_B; + + // A=5, B=20, C=40. (A,B] directs are seeds (submitter batches dropped); + // (B,C] is the full replay stream. + let mk = |sender, block, marker| StoredSafeInput { + sender, + payload: vec![marker], + block_number: block, + }; + let inputs = vec![ + mk(direct, 10, 0x10), // (A,B] direct → seed + mk(submitter, 15, 0x11), // (A,B] batch → dropped (already in S) + mk(direct, 20, 0x12), // (A,B] direct@B → seed + mk(submitter, 25, 0x13), // (B,C] batch → replay + mk(direct, 30, 0x14), // (B,C] direct → replay + ]; + storage + .append_safe_inputs(40, &inputs, submitter, &protocol) + .expect("seed"); + + let checkpoint = Checkpoint::<()> { + app: (), + executed_safe_block: 5, + checkpoint_nonce: 0, + checkpoint_block: 20, + }; + let (seeds, replay) = + source_fold_inputs(&mut storage, &checkpoint, 40, submitter).expect("source"); + + assert_eq!( + seeds.iter().map(|s| s.inclusion_block).collect::>(), + vec![10, 20], + "seeds are the (A,B] directs; the submitter batch @15 is dropped" + ); + assert_eq!( + replay.iter().map(|r| r.inclusion_block).collect::>(), + vec![25, 30], + "replay is the full (B,C] stream (batch + direct)" + ); + } + + #[test] + fn source_fold_inputs_replay_upper_bound_stays_at_c_when_resync_overshoots() { + // The recovery resync runs to the live head H1 > C, so safe_inputs holds + // directs past C. The fold's replay range must stay (B, C]: a direct in + // (C, H1] must NOT leak into replay (it would be folded into S' AND then + // left undrained for run = double execution — the (C, H1] bug class). Unit + // twin of the recovery e2e, pinning the boundary the bug lived at. + use crate::storage::test_helpers::SENDER_B; + let db = temp_db("source-fold-overshoot"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let protocol = default_protocol_timing(); + let submitter = SENDER_A; + let direct = SENDER_B; + + let mk = |block, marker| StoredSafeInput { + sender: direct, + payload: vec![marker], + block_number: block, + }; + // A=5, B=20, C=40, H1=50: a direct at block 50 is in the (C, H1] overshoot. + let inputs = vec![ + mk(10, 0x10), // (A,B] → seed + mk(30, 0x14), // (B,C] → replay + mk(50, 0x20), // (C,H1] → must NOT appear in replay + ]; + storage + .append_safe_inputs(50, &inputs, submitter, &protocol) + .expect("seed"); + + let checkpoint = Checkpoint::<()> { + app: (), + executed_safe_block: 5, + checkpoint_nonce: 0, + checkpoint_block: 20, + }; + let (seeds, replay) = + source_fold_inputs(&mut storage, &checkpoint, 40, submitter).expect("source"); + + assert_eq!( + seeds.iter().map(|s| s.inclusion_block).collect::>(), + vec![10] + ); + assert_eq!( + replay.iter().map(|r| r.inclusion_block).collect::>(), + vec![30], + "the (C, H1] direct @50 must be excluded from replay (upper bound is C, not H1)" + ); + } +} diff --git a/sequencer/src/runtime/setup_fill.rs b/sequencer/src/runtime/setup_fill.rs new file mode 100644 index 0000000..fa1df84 --- /dev/null +++ b/sequencer/src/runtime/setup_fill.rs @@ -0,0 +1,732 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! First-snapshot DB fills for the `setup` subcommand: the two ways `setup` +//! writes the initial finalized snapshot a freshly-prepared DB needs before +//! `run` will boot. +//! +//! - [`register_genesis_finalized_snapshot`] — plain `setup` (phase A): the +//! genesis app state at nonce 0 / cursor 0 / `B = 0`. +//! - [`fill_recovery_state`] — `setup --recovery`: the folded +//! cockroach-recovered state `S'`, tree anchored at `N'`, cursor past the +//! `<= C` directs already in `S'`. +//! +//! Both are setup-time, called once each from [`super::setup::setup`], and +//! never by a runtime worker — hence their own module, distinct from the +//! worker lifecycle in [`super::workers`]. + +use crate::ingress::inclusion_lane::dump_info::{ + self, CreateDumpDirError, create_dump_dir_with_info, +}; +use crate::runtime::error::{RunError, SetupRecoveryError}; +use sequencer_core::application::Application; + +/// Register the genesis application state as the finalized snapshot. Called +/// once by `setup` (phase A). Idempotent: a re-run with a finalized snapshot +/// already present is a no-op, so the `initial_app` is dropped. +/// +/// The genesis dir is unique-per-attempt so a crash between dump creation and +/// `insert_finalized_dump` doesn't wedge a stale directory on the next +/// `setup`. The genesis checkpoint is born finalized: resume nonce 0, replay +/// cursor 0, `B` = 0 (implicit-genesis inclusion block, matching the row). +pub(crate) fn register_genesis_finalized_snapshot( + initial_app: A, + storage: &mut crate::storage::Storage, + dumps_dir: &std::path::Path, +) -> Result<(), RunError> { + if storage.finalized_dump()?.is_some() { + return Ok(()); + } + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let genesis_dir = dumps_dir.join(format!("genesis-{nanos}")); + create_dump_dir_with_info( + &initial_app, + &genesis_dir, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + l2_tx_index: 0, + promoted_inclusion_block: Some(0), + }, + ) + .map_err(|err| match err { + CreateDumpDirError::App(e) => RunError::from(e), + CreateDumpDirError::Io(e) => RunError::from(e), + })?; + storage.insert_finalized_dump(&genesis_dir, 0, 0)?; + Ok(()) +} + +/// Fill the freshly-wiped DB with the cockroach-recovered state, the recovery +/// analog of [`register_genesis_finalized_snapshot`]. Given +/// the folded application state `S'`, the resume batch nonce `N'`, and the +/// post-flush stopping block `C`, it leaves the DB in exactly the shape `run`'s +/// startup expects: a finalized snapshot of `S'`, a batch tree rooted at `N'`, +/// and a replay cursor past every `≤ C` direct (already executed inside `S'`). +/// +/// Crash-before-marker re-entry is **fail-loud**, not blindly idempotent. Only a +/// *completed* fill (finalized snapshot present — the last write) re-runs as a +/// safe no-op. Any other existing root tip is a crashed mid-fill and is refused: +/// * a *different* `N'` (a different checkpoint, or the same one after `C` +/// advanced with more accepted `(B, C]` **batches**) would re-anchor the +/// tree while leaving the old root tip in place, silently breaking I16 — +/// [`SetupRecoveryError::PartialRecoveryMismatch`]; +/// * the *same* `N'` with no finalized snapshot is a crash between step 2 and +/// step 4. A re-sync may have advanced `C` with new **directs** (which leave +/// `N'` unchanged) that resuming would leave unsequenced, so the snapshot +/// cursor lags `S'` and `run` would drain + execute them a second time — +/// [`SetupRecoveryError::PartialRecoveryIncomplete`]. +/// +/// In both cases the operator wipes and re-runs (the one-shot recovery model). +/// +/// Order matters and differs from genesis: the root tip is opened *before* the +/// finalized snapshot, because the snapshot's `l2_tx_index` must equal the +/// global replay head *after* the tip's first frame has sequenced the `(*, C]` +/// directs. `run`'s catch-up replays `offset > l2_tx_index`, so those directs +/// (offsets below the head) are skipped — they are already in `S'`, while +/// on-chain `run`'s first batch (frame `safe_block ≥ C`) drains them once. +pub(crate) fn fill_recovery_state( + recovered_app: A, + resume_nonce: u64, + // `C`, the post-flush stop block; recorded as the snapshot's + // `promoted_inclusion_block` and the recovery tip frame's safe block. + stop_block: u64, + storage: &mut crate::storage::Storage, + dumps_dir: &std::path::Path, +) -> Result<(), RunError> { + // Re-entry guard (rationale + the strict one-shot model are in the + // docstring). The tip is opened before the snapshot, so an existing root tip + // means a prior attempt: only a *completed* fill (finalized snapshot present) + // is a safe no-op; anything else is refused fail-loud. + if let Some(existing) = storage.open_tip_nonce()? { + // Different N' → re-anchoring would orphan the old root tip (I16 break). + if existing != resume_nonce { + return Err(SetupRecoveryError::PartialRecoveryMismatch { + existing_root_nonce: existing, + requested_nonce: resume_nonce, + } + .into()); + } + // Same N', no snapshot → crashed mid-fill; directs that landed since + // would be left unsequenced (cursor lags `S'` → `run` double-drains). + if storage.finalized_dump()?.is_none() { + return Err(SetupRecoveryError::PartialRecoveryIncomplete { + root_nonce: existing, + } + .into()); + } + return Ok(()); + } + // No tip. A fresh fill — or anchor-only residue from a crash before step 2 — + // proceeds below (re-anchoring + opening the tip completes it). But a + // finalized snapshot with *no* root tip is residue from a different + // deployment mode: a plain `setup` that wrote the genesis snapshot and + // crashed before its marker. A completed cockroach fill always has both + // (caught above), so reaching here with a snapshot means recovery is running + // over an un-wiped data dir — silently keeping it would mark setup complete + // over genesis instead of the folded `(S', N')`. Refuse (same fail-loud + // one-shot model as the partial-recovery guards above). + if let Some(finalized) = storage.finalized_dump()? { + return Err(SetupRecoveryError::RecoveryOverResidualSnapshot { + existing_finalized_block: finalized.inclusion_block, + } + .into()); + } + // 1. Anchor the tree at N' so the single parentless root carries it. + storage.set_batch_tree_anchor(resume_nonce)?; + // 2. Open the recovery root tip at N' (parentless, via the anchor), at frame + // safe block `C`, draining only the `≤ C` directs the fold folded into + // `S'` into its first frame's leading range — sequenced but not executed. + // This advances the next-undrained cursor past them so the resumed lane + // never re-leads them. Directs the resync pulled in past `C` (the resync + // runs to the live safe head `H1`, normally `> C`) stay undrained: `run`'s + // lane leads and executes them exactly once as the frontier advances + // `C → H1`. (Draining them here would skip them on catch-up while `S'` + // never executed them — a divergence.) + storage.open_recovery_tip(stop_block)?; + // 3. The finalized snapshot's replay cursor = the global valid replay head + // AFTER step 2's sequencing. + let head = storage.valid_ordered_l2_tx_head()?; + // 4. Register S' as the finalized snapshot at block C (file-first). Unique + // per attempt so a crash before the DB row leaves only a swept orphan. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let recovery_dir = dumps_dir.join(format!("recovery-{nanos}")); + create_dump_dir_with_info( + &recovered_app, + &recovery_dir, + &dump_info::DumpInfo::at_recovery(resume_nonce, head, stop_block), + ) + .map_err(|err| match err { + CreateDumpDirError::App(e) => RunError::from(e), + CreateDumpDirError::Io(e) => RunError::from(e), + })?; + storage.insert_finalized_dump(&recovery_dir, stop_block, head)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ingress::inclusion_lane::{InclusionLane, InclusionLaneConfig, InclusionLaneError}; + use crate::runtime::shutdown::ShutdownSignal; + use crate::runtime::test_support::SweepTestApp; + use crate::storage::Storage; + use crate::storage::test_helpers::temp_db; + use alloy_primitives::{Address, U256}; + use app_core::application::{WalletApp, WalletConfig}; + use std::time::Duration; + + #[test] + fn fill_recovery_state_roots_tree_at_n_prime_and_skips_pre_executed_directs() { + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("fill-recovery"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let submitter = alloy_primitives::Address::repeat_byte(0x99); + let direct = alloy_primitives::Address::repeat_byte(0x22); + let timing = default_protocol_timing(); + + // Sync a safe head at C = 100 with three `≤ C` inputs: a batch (from the + // submitter) and two directs. These stand in for the (A, C] stream the + // fold already folded into S'. + let inputs = vec![ + StoredSafeInput { + sender: submitter, + payload: vec![0x00], + block_number: 10, + }, + StoredSafeInput { + sender: direct, + payload: vec![0xAA], + block_number: 20, + }, + StoredSafeInput { + sender: direct, + payload: vec![0xBB], + block_number: 30, + }, + ]; + storage + .append_safe_inputs(100, &inputs, submitter, &timing) + .expect("sync through C"); + + // Fill at resume nonce N' = 3, C = 100. + let n_prime = 3; + fill_recovery_state(SweepTestApp, n_prime, 100, &mut storage, dumps_dir.path()) + .expect("recovery fill"); + + // The tree is anchored at N' and the single root tip carries it. + assert_eq!(storage.batch_tree_anchor().expect("anchor"), n_prime); + let root_idx = storage + .latest_batch_index() + .expect("idx") + .expect("a root tip exists"); + assert_eq!( + storage.batch_nonce(root_idx).expect("nonce"), + n_prime, + "root tip roots at N' (via the anchor)" + ); + + // All three `≤ C` inputs are sequenced into the root frame, so the + // next-undrained cursor sits past them — the resumed lane never re-leads + // them, and they are already in S'. + assert_eq!( + storage.next_undrained_safe_input_index().expect("cursor"), + 3 + ); + + // Finalized snapshot at C, replay cursor = the post-sequencing head, so + // run's catch-up (offset > l2_tx_index) skips the pre-executed directs. + let finalized = storage + .finalized_dump() + .expect("read") + .expect("finalized snapshot exists"); + assert_eq!(finalized.inclusion_block, 100); + assert_eq!( + finalized.l2_tx_index, 3, + "catch-up starts past the pre-executed (≤C) directs" + ); + + // Idempotent: a re-run (crash before the setup marker) is a no-op, not a + // duplicate-insert error. + fill_recovery_state(SweepTestApp, n_prime, 100, &mut storage, dumps_dir.path()) + .expect("re-run is idempotent"); + assert_eq!(storage.batch_tree_anchor().expect("anchor"), n_prime); + } + + #[test] + fn fill_recovery_state_leaves_post_c_directs_undrained() { + // P1: the post-flush resync runs to the live safe head H1, normally > the + // checkpoint stop block C. The fold folds only `<= C` into S'; directs in + // (C, H1] must stay UNDRAINED so run leads + executes them exactly once. + // Draining them here (the old behavior) would skip them on catch-up while + // S' never executed them — a vanished deposit / divergence. + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("fill-recovery-post-c"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let submitter = alloy_primitives::Address::repeat_byte(0x99); + let direct = alloy_primitives::Address::repeat_byte(0x22); + let timing = default_protocol_timing(); + + // C = 100; the resync reached H1 = 150. Three directs <= C, one at block + // 120 in the (C, H1] window. + let inputs = vec![ + StoredSafeInput { + sender: direct, + payload: vec![0x01], + block_number: 10, + }, + StoredSafeInput { + sender: direct, + payload: vec![0x02], + block_number: 20, + }, + StoredSafeInput { + sender: direct, + payload: vec![0x03], + block_number: 30, + }, + StoredSafeInput { + sender: direct, + payload: vec![0x04], + block_number: 120, + }, + ]; + storage + .append_safe_inputs(150, &inputs, submitter, &timing) + .expect("sync through H1"); + + fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()).expect("fill"); + + // All four inputs are in safe_inputs ... + assert_eq!(storage.safe_input_end_exclusive().expect("end"), 4); + // ... but only the three <= C directs are sequenced; the block-120 direct + // (index 3) stays undrained for run to lead + execute. + assert_eq!( + storage.next_undrained_safe_input_index().expect("cursor"), + 3, + "the (C, H1] direct must remain undrained" + ); + let finalized = storage + .finalized_dump() + .expect("read") + .expect("finalized snapshot"); + assert_eq!( + finalized.l2_tx_index, 3, + "catch-up cursor stops at the <= C directs, not the (C, H1] one" + ); + } + + #[test] + fn fill_recovery_state_drain_boundary_is_inclusive_at_exactly_c() { + // The (C, H1] split must be inclusive at exactly C: a direct at block C + // is folded into S' and drained here; one at C+1 is not. Off-by-one in + // either direction is the same loss/double-execution class as the (C, H1] + // bug, from the boundary. + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("fill-recovery-boundary"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let submitter = alloy_primitives::Address::repeat_byte(0x99); + let direct = alloy_primitives::Address::repeat_byte(0x22); + let timing = default_protocol_timing(); + + // C = 100. Directs straddling it exactly: block 99 (< C), block 100 + // (== C), block 101 (> C). Synced head H1 = 150. + let inputs = vec![ + StoredSafeInput { + sender: direct, + payload: vec![0x01], + block_number: 99, + }, + StoredSafeInput { + sender: direct, + payload: vec![0x02], + block_number: 100, + }, + StoredSafeInput { + sender: direct, + payload: vec![0x03], + block_number: 101, + }, + ]; + storage + .append_safe_inputs(150, &inputs, submitter, &timing) + .expect("sync through H1"); + + fill_recovery_state(SweepTestApp, 0, 100, &mut storage, dumps_dir.path()).expect("fill"); + + // Blocks 99 and 100 (<= C) are drained; block 101 (> C) is not. Inclusive + // at exactly C: cursor sits at index 2, not 1 (would drop the ==C direct) + // and not 3 (would over-drain the >C direct). + assert_eq!( + storage.next_undrained_safe_input_index().expect("cursor"), + 2, + "the block-C direct must be drained; the block-C+1 direct must not" + ); + let finalized = storage + .finalized_dump() + .expect("read") + .expect("finalized snapshot"); + assert_eq!(finalized.l2_tx_index, 2); + } + + #[test] + fn fill_recovery_state_refuses_re_run_with_a_different_nonce() { + // B1: a re-run with a *different* resume nonce (e.g. a different + // checkpoint, or the same one after C advanced) would move the anchor + // while leaving the old root tip — a silent I16 break. It must fail loud. + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("fill-recovery-nonce-mismatch"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let submitter = alloy_primitives::Address::repeat_byte(0x99); + let timing = default_protocol_timing(); + let inputs = vec![StoredSafeInput { + sender: alloy_primitives::Address::repeat_byte(0x22), + payload: vec![0xAA], + block_number: 10, + }]; + storage + .append_safe_inputs(100, &inputs, submitter, &timing) + .expect("sync"); + + // First attempt fills at N' = 3 (opens a root tip carrying nonce 3). + fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()) + .expect("first fill"); + + // Re-run with a DIFFERENT nonce (5): the existing root tip carries 3, so + // the tip-nonce guard fires *before* the finalized short-circuit. + let err = fill_recovery_state(SweepTestApp, 5, 100, &mut storage, dumps_dir.path()) + .expect_err("different-nonce re-run must fail loud"); + assert!( + matches!( + err, + RunError::Bootstrap(crate::runtime::error::BootstrapError::SetupRecovery( + SetupRecoveryError::PartialRecoveryMismatch { + existing_root_nonce: 3, + requested_nonce: 5, + } + )) + ), + "expected PartialRecoveryMismatch, got {err:?}" + ); + } + + #[test] + fn fill_recovery_state_refuses_incomplete_same_nonce_re_run() { + // P1: a crash between opening the recovery root tip (fill step 2) and + // writing the finalized snapshot (step 4) leaves "tip exists, no + // finalized". A same-N' re-run must NOT resume — directs that landed + // since would be left unsequenced, leaving the snapshot cursor behind + // `S'` and double-draining them on `run`. Must fail loud. + // + // We reproduce the EXACT on-disk residue a crashed fill leaves by + // running the real fill writes (anchor + `open_recovery_tip`) and + // stopping before the snapshot — no process crash / test hook needed, + // since the residue is fully determined by which committed storage + // writes happened. + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("fill-recovery-incomplete"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let submitter = alloy_primitives::Address::repeat_byte(0x99); + let direct = alloy_primitives::Address::repeat_byte(0x22); + let timing = default_protocol_timing(); + + // Sync a safe head with one `≤ C` direct, then reproduce a crashed + // mid-fill at N' = 3 with the production writes: anchor set + recovery + // tip opened at C = 100, but no finalized snapshot. + storage + .append_safe_inputs( + 100, + &[StoredSafeInput { + sender: direct, + payload: vec![0xAA], + block_number: 20, + }], + submitter, + &timing, + ) + .expect("sync"); + storage.set_batch_tree_anchor(3).expect("anchor"); + storage + .open_recovery_tip(100) + .expect("open recovery root tip"); + assert!( + storage.finalized_dump().expect("read").is_none(), + "precondition: no finalized snapshot (crashed mid-fill)" + ); + + // A new direct lands during the retry gap (C advances; N' unchanged). + storage + .append_safe_inputs( + 130, + &[StoredSafeInput { + sender: direct, + payload: vec![0xBB], + block_number: 120, + }], + submitter, + &timing, + ) + .expect("resync with a new direct"); + + // Re-run at the SAME N' = 3 must refuse — resuming would leave the new + // direct unsequenced → double-execution on `run`. + let err = fill_recovery_state(SweepTestApp, 3, 130, &mut storage, dumps_dir.path()) + .expect_err("incomplete same-nonce re-run must fail loud"); + assert!( + matches!( + err, + RunError::Bootstrap(crate::runtime::error::BootstrapError::SetupRecovery( + SetupRecoveryError::PartialRecoveryIncomplete { root_nonce: 3 } + )) + ), + "expected PartialRecoveryIncomplete, got {err:?}" + ); + } + + #[test] + fn fill_recovery_state_refuses_over_residual_finalized_snapshot() { + // P2: `setup --recovery` over an un-wiped data dir left by a plain + // `setup` that wrote the genesis finalized snapshot and crashed before + // its marker (finalized snapshot present, NO root tip). A completed + // cockroach fill always has both, so this residue must fail loud — + // silently keeping it would mark setup complete over genesis instead of + // the folded `(S', N')`. + let db = temp_db("fill-recovery-residue"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + + // Plain-setup residue: genesis finalized snapshot, no root tip. + register_genesis_finalized_snapshot(SweepTestApp, &mut storage, dumps_dir.path()) + .expect("genesis snapshot"); + assert!( + storage.latest_batch_index().expect("idx").is_none(), + "precondition: no root tip" + ); + + let err = fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()) + .expect_err("recovery over residual finalized snapshot must fail loud"); + assert!( + matches!( + err, + RunError::Bootstrap(crate::runtime::error::BootstrapError::SetupRecovery( + SetupRecoveryError::RecoveryOverResidualSnapshot { + existing_finalized_block: 0, + } + )) + ), + "expected RecoveryOverResidualSnapshot, got {err:?}" + ); + } + + /// The (C, H1] deposit-window property, end-to-end at the run side: a portal + /// deposit that lands AFTER the flush fixed `C` but within the resync's reach + /// (`H1 > C`) is left UNDRAINED by the recovery fill, then led + executed + /// EXACTLY ONCE by the real inclusion lane as the safe frontier advances + /// `C -> H1` — not lost (drained early / skipped on catch-up) and not + /// double-credited (executed by both the recovery drain and the lane lead). + /// + /// This drives the production lane in-process against a recovered DB — no + /// subprocess, no L1, no test hooks — and replaces the former subprocess + /// e2e. The per-cap unit tests (`..._leaves_post_c_directs_undrained`, + /// `..._drain_boundary_is_inclusive_at_exactly_c`, the fold-source bound) + /// pin each piece; this is the only test that observes the *composition* + /// crediting the deposit exactly once across the setup -> run handoff. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn recovery_then_lane_credits_post_c_deposit_exactly_once() { + use crate::storage::StoredSafeInput; + use crate::storage::test_helpers::default_protocol_timing; + + let db = temp_db("recovery-credited-once"); + let dumps_dir = tempfile::tempdir().expect("dumps dir"); + let timing = default_protocol_timing(); + + // The recovered checkpoint state S' is the real `WalletApp` (only it + // exposes a queryable balance). Its config round-trips through the SSZ + // dump, so the lane reloads a devnet-configured app. + let cfg = WalletConfig::devnet(); + let portal = cfg.erc20_portal_address; + let token = cfg.supported_erc20_token; + let recovered = WalletApp::new(cfg); + let submitter = Address::repeat_byte(0x99); // ingest classifier (not the portal) + let pre_c_direct = Address::repeat_byte(0x22); // non-portal => decode None => inert + let nested_sender = Address::repeat_byte(0x77); // fresh account, zero balance in S' + let deposit_value = U256::from(2_000_000_u64); + + // C = 100; the resync reached H1 = 150. Three `<= C` directs (already + // folded into S'), plus ONE portal USDC deposit at block 120 in the + // (C, H1] window. Deposit payload mirrors `encode_erc20_deposit_payload`: + // token || nested_sender || value_be32. + let deposit_payload = { + let mut p = Vec::with_capacity(20 + 20 + 32); + p.extend_from_slice(token.as_slice()); + p.extend_from_slice(nested_sender.as_slice()); + p.extend_from_slice(deposit_value.to_be_bytes::<32>().as_slice()); + p + }; + let inputs = vec![ + StoredSafeInput { + sender: pre_c_direct, + payload: vec![0x01], + block_number: 10, + }, + StoredSafeInput { + sender: pre_c_direct, + payload: vec![0x02], + block_number: 20, + }, + StoredSafeInput { + sender: pre_c_direct, + payload: vec![0x03], + block_number: 30, + }, + StoredSafeInput { + sender: portal, + payload: deposit_payload, + block_number: 120, + }, + ]; + { + let mut storage = Storage::open(db.path.as_str()).expect("open"); + storage + .append_safe_inputs(150, &inputs, submitter, &timing) + .expect("sync to H1 = 150"); + // Fill at N' = 3, C = 100: drains only the three `<= C` directs into + // the recovery root frame; the deposit (index 3) stays undrained. + fill_recovery_state(recovered, 3, 100, &mut storage, dumps_dir.path()) + .expect("recovery fill"); + + // Preconditions (the (C, H1] cap — covered by the cap tests; pinned + // here so a regression that drained the deposit fails early). + assert_eq!( + storage.next_undrained_safe_input_index().expect("cursor"), + 3, + "the (C, H1] deposit must be left UNDRAINED at index 3" + ); + let finalized = storage + .finalized_dump() + .expect("read") + .expect("finalized S' exists"); + assert_eq!( + finalized.l2_tx_index, 3, + "catch-up starts past the <= C directs" + ); + let s_prime = WalletApp::from_dump(&dump_info::app_prefix(&finalized.dump.prefix)) + .expect("load S'"); + assert_eq!( + s_prime.current_user_balance(nested_sender), + U256::ZERO, + "S' itself must NOT have credited the deposit" + ); + } + + // Drive the REAL run-side inclusion lane against the recovered DB. The + // frontier advance `C -> H1` is already staged (l1_safe_head = 150, the + // recovery tip frame's safe block = 100), so the lane's first iteration + // leads the deposit at index 3 and executes it once. `batch_submitter` + // differs from the portal, so the deposit is not skipped as an own-batch + // input; the short `max_batch_open` forces a batch-close snapshot. + let storage = Storage::open(db.path.as_str()).expect("reopen for lane"); + let config = InclusionLaneConfig { + batch_submitter_address: Address::repeat_byte(0xff), + dumps_dir: dumps_dir.path().to_path_buf(), + max_user_ops_per_chunk: 16, + safe_input_buffer_capacity: 16, + max_batch_open: Duration::from_millis(10), + idle_poll_interval: Duration::from_millis(2), + frontier_min_interval: Duration::ZERO, + }; + let shutdown = ShutdownSignal::default(); + let (_tx, handle) = + InclusionLane::::start(128, shutdown.clone(), storage, config); + + // Observe via a SECOND storage handle (WAL); the lane owns its own. The + // executed-deposit state is externalized only through the pending dump + // the lane writes at batch close. Wait for it to reflect the credit. + let credited = wait_until(Duration::from_secs(5), || { + let mut s = Storage::open(db.path.as_str()).expect("open observer"); + match s.latest_pending_dump().expect("read pending") { + Some(p) => { + WalletApp::from_dump(&dump_info::app_prefix(&p.dump.prefix)) + .expect("load lane snapshot") + .current_user_balance(nested_sender) + == deposit_value + } + None => false, + } + }) + .await; + assert!( + credited, + "the lane must lead + execute the (C, H1] deposit, crediting it once" + ); + + // Exactly once: the drain cursor advanced to 4 (one past the deposit), so + // a restart would not re-lead it; and the balance is the deposit value + // (== once; 0 would be lost, 2x would be double-credited). + { + let mut s = Storage::open(db.path.as_str()).expect("open observer"); + assert_eq!( + s.next_undrained_safe_input_index().expect("cursor"), + 4, + "the deposit's drain cursor must advance exactly one past it" + ); + let dump = s + .latest_pending_dump() + .expect("pending") + .expect("a post-deposit pending dump"); + let app = + WalletApp::from_dump(&dump_info::app_prefix(&dump.dump.prefix)).expect("load"); + assert_eq!( + app.current_user_balance(nested_sender), + deposit_value, + "credited EXACTLY once" + ); + } + + shutdown_lane(&shutdown, handle).await; + } + + async fn wait_until(timeout: Duration, mut predicate: impl FnMut() -> bool) -> bool { + let started = tokio::time::Instant::now(); + while started.elapsed() < timeout { + if predicate() { + return true; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + predicate() + } + + async fn shutdown_lane( + shutdown: &ShutdownSignal, + handle: tokio::task::JoinHandle>, + ) { + shutdown.request_shutdown(); + let joined = tokio::time::timeout(Duration::from_secs(2), handle) + .await + .expect("wait for lane shutdown"); + let result = joined.expect("join lane task"); + assert!(result.is_ok(), "lane should shut down cleanly: {result:?}"); + } +} diff --git a/sequencer/src/runtime/test_support.rs b/sequencer/src/runtime/test_support.rs new file mode 100644 index 0000000..f12a086 --- /dev/null +++ b/sequencer/src/runtime/test_support.rs @@ -0,0 +1,83 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Shared `#[cfg(test)]` fixtures for the runtime modules: a minimal +//! [`Application`] stub plus a dump-layout helper, used by both the worker +//! lifecycle tests ([`super::workers`]) and the setup-fill tests +//! ([`super::setup_fill`]). + +use std::path::Path; + +use crate::ingress::inclusion_lane::dump_info::{self, create_dump_dir_with_info}; +use sequencer_core::application::{AppError, AppOutputs, Application, InvalidReason}; +use sequencer_core::l2_tx::ValidUserOp; +use sequencer_core::user_op::UserOp; + +/// Application stub used in the sweep tests: `create_dump` makes +/// a directory with a marker file inside, `delete_dump` is +/// `remove_dir_all`. The actual marker content is irrelevant — +/// we only care about which directories exist post-sweep. +pub(crate) struct SweepTestApp; + +impl Application for SweepTestApp { + const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn validate_user_op( + &self, + _sender: alloy_primitives::Address, + _user_op: &UserOp, + _current_fee: u16, + ) -> Result<(), InvalidReason> { + Ok(()) + } + fn execute_valid_user_op( + &mut self, + _user_op: &ValidUserOp, + _safe_block: u64, + ) -> Result { + Ok(Vec::new()) + } + fn execute_direct_input( + &mut self, + _input: &sequencer_core::l2_tx::DirectInput, + ) -> Result { + unimplemented!("not used in these tests") + } + fn executed_input_count(&self) -> u64 { + 0 + } + fn last_executed_safe_block(&self) -> u64 { + 0 + } + + fn from_dump(_prefix: &Path) -> Result { + Ok(SweepTestApp) + } + fn create_dump(&self, prefix: &Path) -> Result<(), AppError> { + std::fs::create_dir(prefix)?; + std::fs::write(prefix.join("state"), b"")?; + Ok(()) + } + fn delete_dump(prefix: &Path) -> Result<(), AppError> { + std::fs::remove_dir_all(prefix)?; + Ok(()) + } + fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { + prefix.join("state") + } +} + +/// Mirror of the production dump layout for these tests: structured +/// dir with `info.toml` + the stub app's dump under `state`. +pub(crate) fn create_structured_dump(dump_dir: &std::path::Path) { + create_dump_dir_with_info( + &SweepTestApp, + dump_dir, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + l2_tx_index: 0, + promoted_inclusion_block: None, + }, + ) + .expect("create structured dump"); +} diff --git a/sequencer/src/runtime/workers.rs b/sequencer/src/runtime/workers.rs index e3bc6ce..170d399 100644 --- a/sequencer/src/runtime/workers.rs +++ b/sequencer/src/runtime/workers.rs @@ -28,7 +28,9 @@ use tracing::warn; use crate::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; use crate::http::{self, ApiConfig}; -use crate::ingress::inclusion_lane::{InclusionLane, InclusionLaneConfig, InclusionLaneError}; +use crate::ingress::inclusion_lane::{ + InclusionLane, InclusionLaneConfig, InclusionLaneError, dump_info, dump_info::delete_dump_dir, +}; use crate::l1::reader::{InputReader, InputReaderError}; use crate::l1::submitter::{ BatchPosterConfig, BatchSubmitter, BatchSubmitterConfig, BatchSubmitterError, @@ -58,14 +60,15 @@ pub(crate) enum FirstExit { /// Inputs to [`Workers::spawn`]. Consumed entirely; the caller has nothing /// further to do with these after the call. /// -/// Pure derivations (`db_path`, `domain`, `input_reader.genesis_block()`) are -/// computed inside `spawn` rather than threaded through here. -pub(crate) struct WorkersConfig { - pub app: A, +/// No genesis app instance: `setup` already registered the finalized genesis +/// snapshot, so the lane reloads via `A::from_dump`. The `domain` is built by +/// `run` from the pinned deployment identity. +pub(crate) struct WorkersConfig { pub run_config: RunConfig, pub l1_config: L1Config, pub timing: ProtocolTiming, pub input_reader: InputReader, + pub domain: alloy_sol_types::Eip712Domain, } /// Owns the five worker handles + the shutdown signal that drives all of them. @@ -84,20 +87,19 @@ impl Workers { /// Build the worker configs, spawn each worker, return the owning struct. /// Logs `listening` once the HTTP server is bound. pub(crate) async fn spawn( - cfg: WorkersConfig, + cfg: WorkersConfig, ) -> Result { let WorkersConfig { - app, run_config, l1_config, timing, input_reader, + domain, } = cfg; // Derived values — kept inside `spawn` so `WorkersConfig` stays // minimal and these aren't computed twice in the caller. let db_path = run_config.db_path(); - let domain = run_config.build_domain(); let input_reader_genesis_block = input_reader.genesis_block(); let shutdown = ShutdownSignal::default(); @@ -113,14 +115,18 @@ impl Workers { // 1. Reset stale leases. A crashed previous run may have left // `lease_count > 0` on dumps that aren't being read by // anyone now; without this, GC would skip them forever. - // 2. Ensure a finalized snapshot exists (always-load - // invariant) — cold start uses `app` for the genesis dump. + // 2. Require the finalized snapshot (always-load invariant). `setup` + // registered the genesis snapshot and `run` gated on the + // `setup_complete` marker, so it must be present — a missing one + // is a terminal incomplete-setup, not a cold-start to paper over + // (run holds no genesis app instance). // 3. Ensure an open Tip exists (tip-existence invariant). Opens the // genesis Tip on a fresh DB, no-op otherwise. The lane loads the // head itself after catch-up; this step only establishes the // invariant. Runs after preemptive recovery has synced the safe // head, so the genesis frame's `safe_block` dates to startup (full - // landing budget), not to an earlier, possibly stale view. + // landing budget), not to an earlier, possibly stale view. (This + // is a B-time quantity — it stays in `run`, never in `setup`.) // 4. GC SQLite-side: drop any rows now unreferenced after // promotions or invalidations that finalized just before // the previous shutdown. @@ -128,7 +134,8 @@ impl Workers { // that aren't tracked by SQLite (crash-during-create_dump // or crash-during-GC-after-row-delete artifacts). storage.reset_dump_leases()?; - ensure_finalized_snapshot::(app, &mut storage, &dumps_dir)?; + require_finalized_snapshot(&mut storage)?; + restamp_finalized_promotion(&mut storage)?; storage.ensure_open_tip()?; let gc_removed = snapshot_gc_at_startup::(&mut storage)?; let sweep_removed = sweep_orphan_dumps::(&mut storage, &dumps_dir)?; @@ -142,7 +149,8 @@ impl Workers { QUEUE_CAPACITY, shutdown.clone(), storage, - InclusionLaneConfig::new(l1_config.batch_submitter_address, dumps_dir), + InclusionLaneConfig::new(l1_config.batch_submitter_address, dumps_dir) + .with_max_batch_open(run_config.max_batch_open()), ); // Input reader: produces safe-input rows from L1. @@ -155,8 +163,9 @@ impl Workers { batch_submitter_address: l1_config.batch_submitter_address, start_block: input_reader_genesis_block, confirmation_depth: run_config.batch_submitter_confirmation_depth, - seconds_per_block: run_config.seconds_per_block, + seconds_per_block: run_config.timing.seconds_per_block, long_block_range_error_codes: run_config.long_block_range_error_codes.clone(), + expected_chain_id: l1_config.chain_id, }; let provider = build_batch_submitter_provider(&l1_config)?; let poster = Arc::new(EthereumBatchPoster::new(provider, poster_config)); @@ -189,7 +198,13 @@ impl Workers { ApiConfig::default(), http::SnapshotState { db_path: db_path.clone(), - state_file_in_dump: A::state_file_in_dump, + // The DB row stores the dump *directory*; the app's state + // file lives under its `state` subtree. + state_file_in_dump: |dump_dir| { + A::state_file_in_dump(&crate::ingress::inclusion_lane::dump_info::app_prefix( + dump_dir, + )) + }, }, ) .await?; @@ -235,19 +250,19 @@ impl Workers { detector, shutdown: _, } = self; - let components: [(&'static str, ComponentShutdown); 5] = [ - ("server", Box::pin(wait_for_server_shutdown(server))), - ("inclusion lane", Box::pin(wait_for_lane_shutdown(lane))), + let components: [(WorkerId, ComponentShutdown); 5] = [ + (WorkerId::Server, Box::pin(wait_for_server_shutdown(server))), + (WorkerId::Lane, Box::pin(wait_for_lane_shutdown(lane))), ( - "input reader", + WorkerId::InputReader, Box::pin(wait_for_input_reader_shutdown(reader)), ), ( - "batch submitter", + WorkerId::BatchSubmitter, Box::pin(wait_for_batch_submitter_shutdown(submitter)), ), ( - "danger detector", + WorkerId::DangerDetector, Box::pin(wait_for_danger_detector_shutdown(detector)), ), ]; @@ -259,35 +274,44 @@ impl Workers { // - Signal-driven shutdown: an OS signal triggered shutdown. Wait for // everything to drain; the signal handler's own error (if any) // takes priority over any subsequent component shutdown error. - let (worker_failure, signal_error): (Option<(&'static str, WorkerExit)>, Option) = + let (worker_failure, signal_error): (Option<(WorkerId, WorkerExit)>, Option) = match first_exit { FirstExit::Signal(err) => (None, err), FirstExit::Worker(exit) => { - let name = exit.component_name(); - (Some((name, exit)), None) + let id = exit.worker_id(); + (Some((id, exit)), None) } }; if let Some((failed, primary_exit)) = worker_failure { - for (name, fut) in components { - if name == failed { + for (id, fut) in components { + if id == failed { // Drop the primary's future without awaiting — its task // is already done (it's what tripped the select), and // we'll surface its error directly below. drop(fut); continue; } - log_cleanup_result(name, fut.await); + log_cleanup_result(id.label(), fut.await); } return Err(RunError::Worker(primary_exit)); } - // Signal path: short-circuit on first shutdown error. + // Signal path: await EVERY component so each worker's JoinHandle is + // joined and its task fully drains. A `break` here would drop the + // remaining components' futures un-awaited, which DETACHES those tasks + // (only `JoinHandle::abort()` cancels a dropped handle) — they'd be + // killed mid-drain at runtime teardown, the exact abrupt-write case the + // startup snapshot hygiene (sweep/gc/re-stamp) exists to clean up after. + // Keep the first error to surface; log every error (the signal handler's + // own error, if any, still takes priority below). let mut shutdown_error: Option = None; - for (_, fut) in components { + for (id, fut) in components { if let Err(e) = fut.await { - shutdown_error = Some(e); - break; + warn!(component = id.label(), error = %e, "component errored during signal-driven shutdown"); + if shutdown_error.is_none() { + shutdown_error = Some(e); + } } } match (signal_error, shutdown_error) { @@ -298,16 +322,40 @@ impl Workers { } } +/// Stable identity of each long-lived worker. The `finish` worker-failure path +/// skips the already-exited worker by matching on this enum, not on a label +/// string that could silently drift from the component-array order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WorkerId { + Server, + Lane, + InputReader, + BatchSubmitter, + DangerDetector, +} + +impl WorkerId { + /// Human-readable label for logs, matching the `Workers::finish` list. + fn label(self) -> &'static str { + match self { + WorkerId::Server => "server", + WorkerId::Lane => "inclusion lane", + WorkerId::InputReader => "input reader", + WorkerId::BatchSubmitter => "batch submitter", + WorkerId::DangerDetector => "danger detector", + } + } +} + impl WorkerExit { - /// Human-readable component label, matching the names used in the - /// `Workers::finish` component list. - fn component_name(&self) -> &'static str { + /// Which worker produced this exit. + fn worker_id(&self) -> WorkerId { match self { - WorkerExit::Server(_) => "server", - WorkerExit::Lane(_) => "inclusion lane", - WorkerExit::InputReader(_) => "input reader", - WorkerExit::BatchSubmitter(_) => "batch submitter", - WorkerExit::DangerDetector(_) => "danger detector", + WorkerExit::Server(_) => WorkerId::Server, + WorkerExit::Lane(_) => WorkerId::Lane, + WorkerExit::InputReader(_) => WorkerId::InputReader, + WorkerExit::BatchSubmitter(_) => WorkerId::BatchSubmitter, + WorkerExit::DangerDetector(_) => WorkerId::DangerDetector, } } } @@ -435,41 +483,39 @@ fn log_cleanup_result(component: &str, result: Result<(), WorkerExit>) { } } +// Built once at worker spawn (sync, raw `create_signer_provider`). The submitter +// is long-lived, so a one-shot spawn-time chain-id check would go stale; the +// keyed-write guard instead lives in `EthereumBatchPoster::submit_batches`, +// which re-confirms the chain id immediately before every productive send. fn build_batch_submitter_provider(l1: &L1Config) -> Result { crate::l1::provider::create_signer_provider(&l1.eth_rpc_url, &l1.batch_submitter_private_key) .map_err(std::io::Error::other) } -/// Ensure a finalized snapshot exists before the lane starts. -/// -/// - **Warm start** (finalized snapshot exists): no-op. The lane -/// loads its Application via `from_dump` on its own thread. -/// - **Cold start** (no snapshot yet): consume `initial_app`, write -/// its state as the genesis dump, register it as finalized. The -/// instance is dropped at the end of this function; the lane -/// reloads from the dump it just wrote. -/// -/// Either way, by the time this returns there's a finalized dump -/// the lane can `from_dump` against. -fn ensure_finalized_snapshot( - initial_app: A, - storage: &mut crate::storage::Storage, - dumps_dir: &std::path::Path, -) -> Result<(), RunError> { - if storage.finalized_dump()?.is_some() { - // Warm start: drop `initial_app` and let the lane reload. - return Ok(()); +/// Require the finalized snapshot the lane will `from_dump` against. `setup` +/// registers the genesis snapshot and `run` gates on the `setup_complete` +/// marker, so by the time the lane starts the snapshot must exist. A missing +/// one means the DB's setup is incomplete/corrupt — terminal +/// `SetupNotComplete` (re-run `setup`), not a cold-start to silently heal. +fn require_finalized_snapshot(storage: &mut crate::storage::Storage) -> Result<(), RunError> { + if storage.finalized_dump()?.is_none() { + return Err(RunError::Bootstrap( + crate::runtime::error::BootstrapError::SetupNotComplete, + )); + } + Ok(()) +} + +/// Re-stamp `B` into the finalized dump's `info.toml` from the +/// authoritative DB row. Idempotent; closes the crash window between a +/// promotion's commit and the lane's in-place stamp. +fn restamp_finalized_promotion(storage: &mut crate::storage::Storage) -> Result<(), RunError> { + if let Some(finalized) = storage.finalized_dump()? { + dump_info::stamp_promoted_inclusion_block( + &finalized.dump.prefix, + finalized.inclusion_block, + )?; } - // Cold start: the genesis prefix is unique-per-attempt so a crash - // between `create_dump` and `insert_finalized_dump` doesn't wedge - // a stale directory on the next startup. - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let genesis_prefix = dumps_dir.join(format!("genesis-{nanos}")); - initial_app.create_dump(&genesis_prefix)?; - storage.insert_finalized_dump(&genesis_prefix, 0, 0)?; Ok(()) } @@ -482,7 +528,7 @@ fn snapshot_gc_at_startup( ) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { - if let Err(err) = A::delete_dump(&row.prefix) { + if let Err(err) = delete_dump_dir::(&row.prefix) { tracing::warn!( error = %err, prefix = ?row.prefix, @@ -493,17 +539,18 @@ fn snapshot_gc_at_startup( Ok(removed.len()) } -/// Walk `dumps_dir` and `Application::delete_dump` anything that -/// isn't in `Storage::list_dump_rows`. Catches: +/// Walk `dumps_dir` and delete any dump directory that isn't in +/// `Storage::list_dump_rows`. Catches: /// -/// - **crash-during-create_dump**: file/directory exists on disk but -/// no SQLite row was ever written for it. -/// - **crash-during-GC**: SQLite row was deleted but -/// `Application::delete_dump` either wasn't called or failed. +/// - **crash-during-create**: a dump dir exists on disk (possibly +/// without its app subtree or `info.toml`) but no SQLite row was +/// ever written for it. +/// - **crash-during-GC**: SQLite row was deleted but the filesystem +/// delete either wasn't reached or failed. /// /// Filesystem-only — no SQLite writes here. Failures log and /// continue (the next startup retries). The post-`ensure_finalized` -/// ordering matters: the genesis dump's prefix is in +/// ordering matters: the genesis dump's dir is in /// `list_dump_rows` by the time this runs, so we never delete it. fn sweep_orphan_dumps( storage: &mut crate::storage::Storage, @@ -521,7 +568,7 @@ fn sweep_orphan_dumps( if known.contains(&path) { continue; } - match A::delete_dump(&path) { + match delete_dump_dir::(&path) { Ok(()) => removed += 1, Err(err) => { tracing::warn!( @@ -585,60 +632,9 @@ mod tests { // those are the genesis dump, finalized, and any pending the // lane is still working with. + use crate::runtime::test_support::{SweepTestApp, create_structured_dump}; use crate::storage::Storage; use crate::storage::test_helpers::temp_db; - use sequencer_core::application::{AppError, AppOutputs, Application, InvalidReason}; - use sequencer_core::l2_tx::ValidUserOp; - use sequencer_core::user_op::UserOp; - use std::path::Path; - - /// Application stub used in the sweep tests: `create_dump` makes - /// a directory with a marker file inside, `delete_dump` is - /// `remove_dir_all`. The actual marker content is irrelevant — - /// we only care about which directories exist post-sweep. - struct SweepTestApp; - - impl Application for SweepTestApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; - fn current_user_nonce(&self, _sender: alloy_primitives::Address) -> u32 { - 0 - } - fn current_user_balance( - &self, - _sender: alloy_primitives::Address, - ) -> alloy_primitives::U256 { - alloy_primitives::U256::ZERO - } - fn validate_user_op( - &self, - _sender: alloy_primitives::Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result<(), InvalidReason> { - Ok(()) - } - fn execute_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - ) -> Result { - Ok(Vec::new()) - } - fn from_dump(_prefix: &Path) -> Result { - Ok(SweepTestApp) - } - fn create_dump(&self, prefix: &Path) -> Result<(), AppError> { - std::fs::create_dir(prefix)?; - std::fs::write(prefix.join("state"), b"")?; - Ok(()) - } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { - prefix.join("state") - } - } #[test] fn sweep_orphan_dumps_removes_directories_not_in_storage() { @@ -648,16 +644,18 @@ mod tests { // Tracked dump (in SQLite). let tracked = dumps_dir.path().join("tracked"); - SweepTestApp.create_dump(&tracked).expect("tracked"); + create_structured_dump(&tracked); storage .insert_finalized_dump(&tracked, 0, 0) .expect("register tracked"); - // Two orphans (NOT in SQLite). + // Two orphans (NOT in SQLite). One is fully formed; the other + // mimics a crash between dir creation and the app dump (no + // `state` subtree) — the sweep must remove both. let orphan_a = dumps_dir.path().join("orphan-a"); let orphan_b = dumps_dir.path().join("orphan-b"); - SweepTestApp.create_dump(&orphan_a).expect("orphan a"); - SweepTestApp.create_dump(&orphan_b).expect("orphan b"); + create_structured_dump(&orphan_a); + std::fs::create_dir(&orphan_b).expect("orphan b dir"); let removed = sweep_orphan_dumps::(&mut storage, dumps_dir.path()).unwrap(); assert_eq!(removed, 2); @@ -685,8 +683,8 @@ mod tests { // Two dumps: superseded + finalized. let superseded = dumps_dir.path().join("superseded"); let finalized = dumps_dir.path().join("finalized"); - SweepTestApp.create_dump(&superseded).expect("superseded"); - SweepTestApp.create_dump(&finalized).expect("finalized"); + create_structured_dump(&superseded); + create_structured_dump(&finalized); storage .insert_pending_dump(&superseded, 0, 0) .expect("pending 0"); diff --git a/sequencer/src/storage/egress.rs b/sequencer/src/storage/egress.rs index d04752e..d182d17 100644 --- a/sequencer/src/storage/egress.rs +++ b/sequencer/src/storage/egress.rs @@ -17,13 +17,17 @@ use sequencer_core::l2_tx::SequencedL2Tx; impl Storage { /// Load a page of ordered L2 transactions starting after the given offset. - /// Returns `(db_offset, tx)` pairs. Callers should track `db_offset` of the - /// last item as their cursor, not increment a counter. + /// Returns `(db_offset, tx, frame_safe_block)` triples — the third element + /// is the covering frame's `safe_block`. Catch-up replay feeds it to + /// `execute_valid_user_op` so the app's safe-block clock advances exactly + /// as it did live (directs use their own `block_number` instead; the feed + /// ignores it). Callers should track `db_offset` of the last item as their + /// cursor, not increment a counter. pub fn ordered_l2_txs_page_from( &mut self, offset: u64, limit: usize, - ) -> Result> { + ) -> Result> { if limit == 0 { return Ok(Vec::new()); } @@ -40,7 +44,8 @@ impl Storage { CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number, + f.safe_block FROM valid_sequenced_l2_txs s LEFT JOIN user_ops u ON u.batch_index = s.batch_index @@ -66,7 +71,10 @@ impl Storage { row.get(5)?, row.get(6)?, ); - Ok((i64_to_u64(db_offset), tx)) + // Non-NULL for every sequenced row: the frames row is inserted + // when the frame opens, before anything is sequenced into it. + let frame_safe_block: i64 = row.get(7)?; + Ok((i64_to_u64(db_offset), tx, i64_to_u64(frame_safe_block))) })?; rows.collect::>>() } diff --git a/sequencer/src/storage/ingress.rs b/sequencer/src/storage/ingress.rs index 1e67a9c..57deb61 100644 --- a/sequencer/src/storage/ingress.rs +++ b/sequencer/src/storage/ingress.rs @@ -97,6 +97,26 @@ impl Storage { }) } + /// Open the cockroach-recovery root tip: a fresh anchored tip (`batch_index` + /// 0, parentless, nonce = the batch-tree anchor `N'`) whose first frame sits + /// at the checkpoint stop block `C` and leads exactly the directs the fold + /// folded into `S'` — those with `block_number <= C`. + /// + /// Distinct from [`Storage::ensure_open_tip`] / [`open_fresh_tip_in_tx`], + /// which drain the **whole** synced table at the live safe head. That is + /// correct for genesis, but wrong here: `setup --recovery` resyncs to the + /// live safe head `H1`, which is normally **past** `C`, and the fold only + /// folded `<= C` into `S'`. Draining `(C, H1]` into this tip would sequence + /// those directs as "already executed" (advancing the cursor / snapshot + /// `l2_tx_index` past them) even though `S'` never executed them — they would + /// then be skipped by catch-up and never re-led, vanishing from local state + /// while the scheduler drains them on-chain (divergence). Capping the drain + /// at `C` leaves `(C, H1]` undrained, so `run`'s lane leads and executes them + /// exactly once as the safe frontier advances `C -> H1`. + pub fn open_recovery_tip(&mut self, stop_block: u64) -> Result<()> { + self.write(|tx| open_recovery_tip_in_tx(tx, stop_block)) + } + /// Snapshot the current L1 view: safe block + exclusive safe-input cursor. /// The lane uses this to decide whether to advance. /// @@ -282,23 +302,30 @@ impl Storage { /// fails, promotion wedges on `QueryReturnedNoRows` forever" gap. /// /// `nonce` is the closing batch's nonce (assigned at open, so known - /// before the seal). The snapshot's `l2_tx_index` is read inside the - /// tx as the global valid replay head — correct even when the - /// closing batch is empty. + /// before the seal). `l2_tx_index` is the global valid replay head + /// the caller read when writing the dump's `info.toml` — correct + /// even when the closing batch is empty. pub fn close_frame_and_batch_with_pending_dump( &mut self, head: &mut WriteHead, next_safe_block: u64, - dump_prefix: &Path, + dump_dir: &Path, nonce: u64, + l2_tx_index: u64, ) -> Result<()> { let (next_batch_index, now_ms, policy) = self.write(|tx| { - // Read the replay head before sealing so the snapshot records - // the global valid head as of the close. - let l2_tx_index = valid_ordered_l2_tx_head(tx)?; + // The lane is the single writer and nothing sequences between + // the caller's head read and this close, so the head cannot + // have moved. Assert it: the snapshot row and the dump's + // info.toml must record the same cursor. + let head_now = valid_ordered_l2_tx_head(tx)?; + assert_eq!( + head_now, l2_tx_index, + "replay head moved between dump creation and batch close" + ); let (next_batch_index, now_ms, policy) = seal_and_open_next_batch(tx, head.batch_index, next_safe_block)?; - insert_pending_dump_in(tx, dump_prefix, nonce, l2_tx_index)?; + insert_pending_dump_in(tx, dump_dir, nonce, l2_tx_index)?; Ok((next_batch_index, now_ms, policy)) })?; head.move_to_next_batch( @@ -373,20 +400,78 @@ pub(super) fn open_fresh_tip_in_tx(tx: &Transaction<'_>) -> Result<()> { })? .map(i64_to_u64); let batch_index_opt = if table_empty { Some(0) } else { None }; - let leading_direct_range = SafeInputRange::new( - next_undrained_safe_input_index_in(tx)?, - query_latest_safe_input_index_exclusive(tx)?, - ); - insert_tip_rows( + insert_draining_tip( tx, batch_index_opt, parent, safe_block, - leading_direct_range, - )?; + query_latest_safe_input_index_exclusive(tx)?, + ) +} + +/// The shared tail of [`open_fresh_tip_in_tx`] and [`open_recovery_tip_in_tx`]: +/// open the single valid Tip draining `[next_undrained, drain_upper)`, framed at +/// `safe_block`, with the given lineage. The two callers differ only in +/// `drain_upper` — the live safe-input head on the fresh path vs the `<= C` cap +/// on the recovery path (the load-bearing `(C, H1]` difference) — plus +/// `safe_block` and lineage, which they pass explicitly so the cap rule stays +/// visible at each call site rather than hidden in one branch. +fn insert_draining_tip( + tx: &Transaction<'_>, + batch_index: Option, + parent: Option, + safe_block: u64, + drain_upper: u64, +) -> Result<()> { + let leading_direct_range = + SafeInputRange::new(next_undrained_safe_input_index_in(tx)?, drain_upper); + insert_tip_rows(tx, batch_index, parent, safe_block, leading_direct_range)?; Ok(()) } +/// See [`Storage::open_recovery_tip`]. Like [`open_fresh_tip_in_tx`] but the +/// frame's safe block is the checkpoint stop block `C` and the leading drain +/// range is capped at the `<= C` directs (not the whole synced table), so the +/// `(C, H1]` directs the resync pulled past `C` stay undrained for `run`. +fn open_recovery_tip_in_tx(tx: &Transaction<'_>, stop_block: u64) -> Result<()> { + debug_assert!( + load_current_write_head(tx)?.is_none(), + "recovery tip opened over existing open state" + ); + // Fresh DB after wipe: batch_index 0, parentless (rooted at the anchor via + // `compute_next_nonce(None)` -> `N'`); drain capped at the `<= C` safe + // inputs. + // + // That range is sender-unfiltered: it includes the `<= C` batch-submitter + // rows alongside the user directs, exactly as the fresh / genesis tip drains + // its whole `[next_undrained, latest)` span. Correct because the leading + // range is *sequenced, not executed* — it only advances the replay cursor so + // `run`'s catch-up (`offset > l2_tx_index`) skips the rows already in `S'`. + // The `sender != batch_submitter` drop belongs to the *fold's* seed filter + // (those batches were folded into `S'` as batches, not directs); it is not a + // cursor concern, so it deliberately does not reappear here. + insert_draining_tip( + tx, + Some(0), + None, + stop_block, + safe_input_index_exclusive_through_block_in(tx, stop_block)?, + ) +} + +/// Exclusive `safe_input_index` boundary separating directs at `block_number <= +/// block` from those past it. `safe_input_index` is dense and assigned in +/// block-ascending ingest order, so the `<= block` rows occupy `[0, count)` and +/// this count is the boundary. +fn safe_input_index_exclusive_through_block_in(tx: &Transaction<'_>, block: u64) -> Result { + let boundary: i64 = tx.query_row( + "SELECT COUNT(*) FROM safe_inputs WHERE block_number <= ?1", + params![u64_to_i64(block)], + |row| row.get(0), + )?; + Ok(i64_to_u64(boundary)) +} + /// `MAX(safe_input_index) + 1` over the valid drained rows (or 0 if none), /// inside `tx`. The cursor rewinds when a batch is invalidated, so a recovery /// batch re-drains the same range its invalidated predecessor was working from. @@ -417,7 +502,15 @@ fn seal_and_open_next_batch( // Batch policy is sampled here: the derived fee is committed to the newly // opened frame, and the batch size target is stored on the write head. let policy = query_batch_policy(tx)?; - seal_batch(tx, closing_batch_index, now_ms)?; + // Hash-at-seal (review R2): encode the closing batch's wire bytes via + // the same path the submitter uses and stamp their keccak256 on the row, + // atomically with the seal. The content-identity check later compares + // accepted L1 landings against this hash. + let nonce = super::snapshot_dumps::batch_nonce_in(tx, closing_batch_index)?; + let frames = super::l1_submission::load_batch_frames_in(tx, closing_batch_index)?; + let encoded = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { nonce, frames }); + let payload_hash = alloy_primitives::keccak256(&encoded); + seal_batch(tx, closing_batch_index, now_ms, &payload_hash.0)?; let next_batch_index = insert_new_batch(tx, None, Some(closing_batch_index), now_ms)?; insert_open_frame( tx, diff --git a/sequencer/src/storage/l1_inputs.rs b/sequencer/src/storage/l1_inputs.rs index 97344e9..fc4b3f2 100644 --- a/sequencer/src/storage/l1_inputs.rs +++ b/sequencer/src/storage/l1_inputs.rs @@ -17,7 +17,7 @@ use super::queries::{ query_latest_safe_input_index_exclusive, }; use super::safe_accepted_batches::populate_safe_accepted_batches; -use super::{DeploymentIdentity, StoredSafeInput}; +use super::{DeploymentIdentity, FrontierMode, StoredSafeInput}; use sequencer_core::protocol::ProtocolTiming; impl Storage { @@ -35,6 +35,85 @@ impl Storage { current_safe_block_timestamp(&self.conn) } + /// First batch-submitter `safe_inputs` row strictly past `after_block`, by + /// ascending `safe_input_index` — returns `(safe_input_index, block_number)` + /// or `None`. Read-only; `setup`'s detection gate uses it + /// to find any previous-instance batch past the checkpoint block. + /// + /// Queries the reader-synced `safe_inputs` table rather than issuing its own + /// `get_logs`, so it inherits the reader's F5 completeness guarantees (a + /// per-app index contiguity check plus a `getNumberOfInputs` count witness): + /// the reader refuses to persist an incomplete `get_logs` response, so the + /// synced table is complete through the safe head. Do **not** replace this + /// with a fresh log scan, which would bypass that protection. Detection runs + /// after `setup`'s initial sync has populated `safe_inputs` up to the safe + /// head; + /// because step 1 has already confirmed the wallet nonce is settled + /// (nothing of ours sits unsafe), querying the synced safe inputs is + /// equivalent to scanning `(after_block, safe]`. + pub fn first_batch_submitter_input_after_block( + &mut self, + batch_submitter: Address, + after_block: u64, + ) -> Result> { + self.conn + .query_row( + "SELECT safe_input_index, block_number FROM safe_inputs \ + WHERE sender = ?1 AND block_number > ?2 \ + ORDER BY safe_input_index ASC LIMIT 1", + params![batch_submitter.as_slice(), u64_to_i64(after_block)], + |row| { + let index: i64 = row.get(0)?; + let block: i64 = row.get(1)?; + Ok((i64_to_u64(index), i64_to_u64(block))) + }, + ) + .optional() + } + + /// All `safe_inputs` rows whose `block_number` is in `(after_block, + /// through_block]`, ordered by `safe_input_index` ascending — i.e. L1 + /// inclusion order. The recovery fold sources + /// its seeds from `(A, B]` and its replay stream from `(B, C]` through this: + /// half-open on the lower bound (directs at block `B` belong to the fridge, + /// batches at `B` are already in `S` — open-question 1 boundary convention). + /// + /// Returns both senders and directs; the caller classifies (a batch iff + /// `sender == batch_submitter`) and drops batches from the `(A, B]` seed set. + /// Read-only; queries the reader-synced table rather than a fresh log scan, + /// inheriting the reader's F5 completeness guarantees (see + /// [`Storage::first_batch_submitter_input_after_block`]). + pub fn safe_inputs_in_block_range( + &mut self, + after_block: u64, + through_block: u64, + ) -> Result> { + const SQL: &str = "SELECT sender, payload, block_number FROM safe_inputs \ + WHERE block_number > ?1 AND block_number <= ?2 \ + ORDER BY safe_input_index ASC"; + let mut stmt = self.conn.prepare_cached(SQL)?; + let rows = stmt.query_map( + params![u64_to_i64(after_block), u64_to_i64(through_block)], + |row| { + Ok(( + row.get::<_, Vec>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, i64>(2)?, + )) + }, + )?; + let mut out = Vec::new(); + for row in rows { + let (sender, payload, block_number) = row?; + out.push(StoredSafeInput { + sender: Address::from_slice(sender.as_slice()), + payload, + block_number: i64_to_u64(block_number), + }); + } + Ok(out) + } + /// Atomically: insert `inputs` (assigned contiguous indexes starting from /// the current MAX+1), advance `l1_safe_head.block_number` to `safe_block`, /// stamp `synced_at_ms` as the wall-clock time when the safe frontier @@ -62,6 +141,7 @@ impl Storage { inputs, batch_submitter, timing, + FrontierMode::Populate, ) } @@ -69,6 +149,10 @@ impl Storage { /// of `safe_block`. Production input-reader code should use this path; /// the shorter helper exists for tests that only need a fresh synthetic /// safe head. + /// + /// `frontier` gates the `safe_accepted_batches` update — see + /// [`FrontierMode`]. Everything except `setup --recovery`'s interim syncs + /// uses [`FrontierMode::Populate`]. pub fn append_safe_inputs_with_timestamp( &mut self, safe_block: u64, @@ -76,6 +160,7 @@ impl Storage { inputs: &[StoredSafeInput], batch_submitter: Address, timing: &ProtocolTiming, + frontier: FrontierMode, ) -> Result<()> { self.write(|tx| { if let Some(current) = current_safe_block(tx)? { @@ -110,7 +195,10 @@ impl Storage { return Err(rusqlite::Error::StatementChangedRows(changed)); } - populate_safe_accepted_batches(tx, batch_submitter, timing) + if matches!(frontier, FrontierMode::Populate) { + populate_safe_accepted_batches(tx, batch_submitter, timing)?; + } + Ok(()) }) } @@ -172,6 +260,37 @@ impl Storage { Ok(identity) }) } + + /// Record that `setup` finished. This is `setup`'s LAST write — after + /// identity is pinned, the initial L1 sync is durable, and the genesis + /// finalized snapshot is registered. + /// Idempotent: re-running `setup` on an already-complete DB leaves the + /// original `completed_at_ms` untouched. + pub fn mark_setup_complete(&mut self) -> Result<()> { + self.write(|tx| { + // Deliberate idempotency (re-running `setup` is legitimate), not + // silent absorption: keep the first completion timestamp. + tx.execute( + "INSERT INTO setup_complete (singleton_id, completed_at_ms) \ + VALUES (0, ?1) \ + ON CONFLICT(singleton_id) DO NOTHING", + params![now_unix_ms()], + )?; + Ok(()) + }) + } + + /// Whether `setup` has completed on this DB. `run` refuses to boot when + /// this is `false` — the marker absent means either setup never ran or it + /// crashed midway, both of which require `setup` (re-)run, not `run`. + pub fn is_setup_complete(&self) -> Result { + let present: i64 = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM setup_complete WHERE singleton_id = 0)", + [], + |row| row.get(0), + )?; + Ok(present != 0) + } } fn query_deployment_identity(conn: &rusqlite::Connection) -> Result> { @@ -219,7 +338,7 @@ fn insert_safe_inputs_batch( #[cfg(test)] mod tests { use crate::storage::{ - DeploymentIdentity, SafeInputRange, Storage, StoredSafeInput, + DeploymentIdentity, FrontierMode, SafeInputRange, Storage, StoredSafeInput, test_helpers::{SENDER_A, SENDER_B, default_protocol_timing, temp_db}, }; use alloy_primitives::Address; @@ -276,6 +395,155 @@ mod tests { assert!(out.is_empty()); } + #[test] + fn first_batch_submitter_input_after_block_scans_strictly_past() { + let db = temp_db("first-submitter-input-after"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let protocol = default_protocol_timing(); + + // index 0: submitter @5; index 1: non-submitter @12; index 2: submitter @18. + // Payloads are junk (scheduler no-ops on decode) — detection scans the + // raw safe_inputs rows, not the accepted frontier, so acceptance is + // irrelevant: ANY submitter activity past the checkpoint matters. + let inputs = vec![ + StoredSafeInput { + sender: SENDER_A, + payload: vec![0x01], + block_number: 5, + }, + StoredSafeInput { + sender: SENDER_B, + payload: vec![0x02], + block_number: 12, + }, + StoredSafeInput { + sender: SENDER_A, + payload: vec![0x03], + block_number: 18, + }, + ]; + storage + .append_safe_inputs(20, inputs.as_slice(), SENDER_A, &protocol) + .expect("seed safe inputs"); + + // From genesis (0): the earliest submitter input is index 0 @5. + assert_eq!( + storage + .first_batch_submitter_input_after_block(SENDER_A, 0) + .expect("scan"), + Some((0, 5)) + ); + // Strictly past: block 5 is excluded; the non-submitter @12 is skipped; + // next submitter is index 2 @18. + assert_eq!( + storage + .first_batch_submitter_input_after_block(SENDER_A, 5) + .expect("scan"), + Some((2, 18)) + ); + // Past the last submitter input: nothing. + assert_eq!( + storage + .first_batch_submitter_input_after_block(SENDER_A, 18) + .expect("scan"), + None + ); + // A different submitter address never matches SENDER_A's rows. + assert_eq!( + storage + .first_batch_submitter_input_after_block(Address::repeat_byte(0xCC), 0) + .expect("scan"), + None + ); + } + + #[test] + fn safe_inputs_in_block_range_is_half_open_lower_and_ordered() { + let db = temp_db("safe-inputs-block-range"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let protocol = default_protocol_timing(); + + // Blocks 5 (direct), 10 (batch from SENDER_A + direct from SENDER_B), + // 15 (direct). SENDER_A is the batch submitter (identity above). + let inputs = vec![ + StoredSafeInput { + sender: SENDER_B, + payload: vec![0x05], + block_number: 5, + }, + StoredSafeInput { + sender: SENDER_A, + payload: vec![0x10], + block_number: 10, + }, + StoredSafeInput { + sender: SENDER_B, + payload: vec![0x11], + block_number: 10, + }, + StoredSafeInput { + sender: SENDER_B, + payload: vec![0x15], + block_number: 15, + }, + ]; + storage + .append_safe_inputs(15, inputs.as_slice(), SENDER_A, &protocol) + .expect("seed safe inputs"); + + // (5, 15] excludes block 5, includes block 15 — ordered by index. + let mid = storage + .safe_inputs_in_block_range(5, 15) + .expect("range (5,15]"); + let blocks: Vec = mid.iter().map(|i| i.block_number).collect(); + assert_eq!(blocks, vec![10, 10, 15], "half-open lower bound; ascending"); + + // Caller classifies: drop sender == batch_submitter to get the directs + // of the seed range (the (A,B] fridge reconstruction). + let directs: Vec<&[u8]> = mid + .iter() + .filter(|i| i.sender != SENDER_A) + .map(|i| i.payload.as_slice()) + .collect(); + assert_eq!( + directs, + vec![&[0x11u8][..], &[0x15u8][..]], + "batch at block 10 dropped" + ); + + // Empty when the lower bound covers everything. + assert!( + storage + .safe_inputs_in_block_range(15, 15) + .expect("empty") + .is_empty() + ); + // Full span from genesis. + assert_eq!( + storage + .safe_inputs_in_block_range(0, 100) + .expect("all") + .len(), + 4 + ); + } + + #[test] + fn batch_tree_anchor_roundtrips_and_freezes_after_setup() { + let db = temp_db("anchor-roundtrip"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + assert_eq!(storage.batch_tree_anchor().expect("default"), 0); + storage.set_batch_tree_anchor(1200).expect("set anchor"); + assert_eq!(storage.batch_tree_anchor().expect("read back"), 1200); + // Once setup is complete, the public setter aborts too (write-once). + storage.mark_setup_complete().expect("mark complete"); + assert!( + storage.set_batch_tree_anchor(1300).is_err(), + "anchor must be frozen after setup_complete" + ); + assert_eq!(storage.batch_tree_anchor().expect("unchanged"), 1200); + } + #[test] fn new_db_has_no_observed_safe_head() { let db = temp_db("new-db-no-safe-head"); @@ -344,6 +612,48 @@ mod tests { ); } + #[test] + fn setup_complete_marker_absent_until_marked_then_idempotent() { + let db = temp_db("setup-complete-marker"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + + assert!( + !storage + .is_setup_complete() + .expect("read marker on fresh DB"), + "fresh DB has no setup-complete marker" + ); + + storage.mark_setup_complete().expect("mark complete"); + assert!( + storage.is_setup_complete().expect("read marker"), + "marker present after mark_setup_complete" + ); + + let first_ts: i64 = storage + .conn + .query_row( + "SELECT completed_at_ms FROM setup_complete WHERE singleton_id = 0", + [], + |row| row.get(0), + ) + .expect("read completed_at_ms"); + + // Re-running setup is legitimate and must not error or move the + // original timestamp. + storage.mark_setup_complete().expect("mark complete again"); + let second_ts: i64 = storage + .conn + .query_row( + "SELECT completed_at_ms FROM setup_complete WHERE singleton_id = 0", + [], + |row| row.get(0), + ) + .expect("read completed_at_ms"); + assert_eq!(first_ts, second_ts, "idempotent: first timestamp kept"); + assert!(storage.is_setup_complete().expect("read marker")); + } + #[test] fn append_safe_inputs_creates_and_advances_safe_head() { let db = temp_db("append-safe-inputs-creates-safe-head"); @@ -351,7 +661,14 @@ mod tests { let protocol = default_protocol_timing(); storage - .append_safe_inputs_with_timestamp(7, 1234, &[], SENDER_A, &protocol) + .append_safe_inputs_with_timestamp( + 7, + 1234, + &[], + SENDER_A, + &protocol, + FrontierMode::Populate, + ) .expect("record first real safe-head observation"); assert_eq!( storage.current_safe_block().expect("read safe block"), @@ -371,7 +688,14 @@ mod tests { ); storage - .append_safe_inputs_with_timestamp(9, 5678, &[], SENDER_A, &protocol) + .append_safe_inputs_with_timestamp( + 9, + 5678, + &[], + SENDER_A, + &protocol, + FrontierMode::Populate, + ) .expect("advance safe head"); assert_eq!( storage.current_safe_block().expect("read safe block"), diff --git a/sequencer/src/storage/l1_submission.rs b/sequencer/src/storage/l1_submission.rs index 17b7404..1ee6b11 100644 --- a/sequencer/src/storage/l1_submission.rs +++ b/sequencer/src/storage/l1_submission.rs @@ -1,21 +1,22 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Batch-aggregate reads: frontier lookup, per-batch frames + user ops, the -//! catch-up / per-batch replay reader, and the SSZ-encoded pending-batch list -//! the submitter pulls each tick. +//! The submitter's storage half: frontier lookup, per-batch frames + user +//! ops, the catch-up / per-batch replay reader, the SSZ-encoded pending-batch +//! list the submitter pulls each tick — and the one submission-side write, +//! the wallet-nonce watermark (raised before every broadcast, review R1a). //! -//! Despite the historical name, nothing in this file does writes — structural -//! nonces are assigned by the `batches.nonce` trigger at close time (see -//! `ingress`), and `safe_accepted_batches` is maintained by `append_safe_inputs` -//! (see `l1_inputs`). The reads here are shared between the batch submitter -//! (hot-path tick) and the egress replay path (catch-up reader); they live -//! together because they all aggregate at the batch level. +//! Structural nonces are assigned by the `batches.nonce` trigger at close +//! time (see `ingress`), and `safe_accepted_batches` is maintained by +//! `append_safe_inputs` (see `l1_inputs`). The reads here are shared between +//! the batch submitter (hot-path tick) and the egress replay path (catch-up +//! reader); they live together because they all aggregate at the batch level. use rusqlite::{Result, params}; use super::Storage; use super::convert::{i64_to_u16, i64_to_u32, i64_to_u64, u64_to_i64}; +use super::mutations::{batch_tree_anchor_in, set_batch_tree_anchor_in}; use super::queries::{current_safe_block_required, decode_l2_tx_row}; use super::safe_accepted_batches::frontier_nonce; use super::{FrameHeader, PendingBatch, SubmitterFrontier}; @@ -44,6 +45,53 @@ impl Storage { }) } + /// Set the batch-tree anchor nonce (the nonce the parentless root carries). + /// Used by `setup --recovery` to root the rebuilt tree at `N'`. Aborts if + /// `setup_complete` already exists (`trg_batch_tree_anchor_write_once`). + pub fn set_batch_tree_anchor(&mut self, nonce: u64) -> Result<()> { + self.write(|tx| set_batch_tree_anchor_in(tx, nonce)) + } + + /// Read the batch-tree anchor nonce (default 0). + pub fn batch_tree_anchor(&mut self) -> Result { + self.read(|tx| batch_tree_anchor_in(tx)) + } + + /// The highest wallet nonce ever broadcast by this deployment's + /// batch-submitter key, or `None` if nothing was ever broadcast + /// (review R1a — the durable realization of the TLA+ `walletNonce`). + /// The flush reads this as its coverage floor; it never resets. + pub fn wallet_nonce_watermark(&mut self) -> Result> { + use rusqlite::OptionalExtension; + self.read(|tx| { + tx.query_row( + "SELECT watermark FROM wallet_nonce_watermark WHERE singleton_id = 0", + [], + |row| row.get::<_, i64>(0), + ) + .optional() + .map(|v| v.map(i64_to_u64)) + }) + } + + /// Write-before-broadcast (review R1a): durably raise the watermark to + /// cover `nonce` *before* any tx at a nonce `<= nonce` is sent. The + /// commit is power-loss durable (`synchronous=FULL`); a crash between + /// commit and send only over-covers — the flush later no-ops a + /// never-used slot, which is harmless. Monotonic: never lowers. + pub fn raise_wallet_nonce_watermark(&mut self, nonce: u64) -> Result<()> { + self.write(|tx| { + tx.execute( + "INSERT INTO wallet_nonce_watermark (singleton_id, watermark) \ + VALUES (0, ?1) \ + ON CONFLICT(singleton_id) \ + DO UPDATE SET watermark = MAX(watermark, excluded.watermark)", + params![u64_to_i64(nonce)], + )?; + Ok(()) + }) + } + /// Highest valid (non-invalidated) `batch_index`, or `None` if no valid /// batches exist. The open batch is included. pub fn latest_batch_index(&mut self) -> Result> { @@ -55,22 +103,25 @@ impl Storage { Ok(value.map(i64_to_u64)) } + /// The structural nonce of the current open Tip (the single + /// `valid_open_batch`), or `None` if no Tip is open. Used by `setup + /// --recovery` to detect a partially-recovered DB whose root tip carries a + /// different resume nonce than the current attempt (the at-most-one-valid- + /// open-tip index makes this at most one row). + pub fn open_tip_nonce(&mut self) -> Result> { + use rusqlite::OptionalExtension; + let value: Option = self + .conn + .query_row("SELECT nonce FROM valid_open_batch", [], |row| row.get(0)) + .optional()?; + Ok(value.map(i64_to_u64)) + } + /// Frame headers for `batch_index` in `frame_in_batch` order. Reads the /// raw `frames` table — does NOT filter on validity, since callers only /// reach this method after they already know the batch is valid. pub fn frames_for_batch(&mut self, batch_index: u64) -> Result> { - let mut stmt = self.conn.prepare_cached( - "SELECT frame_in_batch, fee, safe_block FROM frames \ - WHERE batch_index = ?1 ORDER BY frame_in_batch ASC", - )?; - let rows = stmt.query_map(params![u64_to_i64(batch_index)], |row| { - Ok(FrameHeader { - frame_in_batch: i64_to_u32(row.get(0)?), - fee: i64_to_u16(row.get(1)?), - safe_block: i64_to_u64(row.get(2)?), - }) - })?; - rows.collect::>>() + frames_for_batch_in(&self.conn, batch_index) } /// Materialize all sequenced L2 txs in one batch (used by the catch-up / @@ -153,44 +204,71 @@ impl Storage { /// Internal helper for [`Self::pending_batches`]; does NOT filter on /// validity — callers only reach this after they know the batch is valid. fn load_batch_frames(&mut self, batch_index: u64) -> Result> { - let frame_headers = self.frames_for_batch(batch_index)?; - let mut frames = Vec::with_capacity(frame_headers.len()); - for header in frame_headers { - let mut stmt = self.conn.prepare_cached( - "SELECT nonce, max_fee, data, sig FROM user_ops \ - WHERE batch_index = ?1 AND frame_in_batch = ?2 \ - ORDER BY pos_in_frame ASC", - )?; - let rows = stmt.query_map( - params![u64_to_i64(batch_index), i64::from(header.frame_in_batch)], - |row| { - Ok(WireUserOp { - nonce: i64_to_u32(row.get(0)?), - max_fee: i64_to_u16(row.get(1)?), - data: row.get(2)?, - signature: row.get(3)?, - }) - }, - )?; - let user_ops: Vec = rows.collect::>()?; - frames.push(BatchFrame { - user_ops, - safe_block: header.safe_block, - fee_price: header.fee, - }); - } - Ok(frames) + load_batch_frames_in(&self.conn, batch_index) } } +fn frames_for_batch_in(conn: &rusqlite::Connection, batch_index: u64) -> Result> { + let mut stmt = conn.prepare_cached( + "SELECT frame_in_batch, fee, safe_block FROM frames \ + WHERE batch_index = ?1 ORDER BY frame_in_batch ASC", + )?; + let rows = stmt.query_map(params![u64_to_i64(batch_index)], |row| { + Ok(FrameHeader { + frame_in_batch: i64_to_u32(row.get(0)?), + fee: i64_to_u16(row.get(1)?), + safe_block: i64_to_u64(row.get(2)?), + }) + })?; + rows.collect::>>() +} + +/// Free-function form so the seal path can encode the closing batch inside +/// its own transaction — the content-identity check's hash-at-seal must come +/// from **the same encode path the submitter uses** (review R2); this +/// function being that single path is load-bearing. +pub(super) fn load_batch_frames_in( + conn: &rusqlite::Connection, + batch_index: u64, +) -> Result> { + let frame_headers = frames_for_batch_in(conn, batch_index)?; + let mut frames = Vec::with_capacity(frame_headers.len()); + for header in frame_headers { + let mut stmt = conn.prepare_cached( + "SELECT nonce, max_fee, data, sig FROM user_ops \ + WHERE batch_index = ?1 AND frame_in_batch = ?2 \ + ORDER BY pos_in_frame ASC", + )?; + let rows = stmt.query_map( + params![u64_to_i64(batch_index), i64::from(header.frame_in_batch)], + |row| { + Ok(WireUserOp { + nonce: i64_to_u32(row.get(0)?), + max_fee: i64_to_u16(row.get(1)?), + data: row.get(2)?, + signature: row.get(3)?, + }) + }, + )?; + let user_ops: Vec = rows.collect::>()?; + frames.push(BatchFrame { + user_ops, + safe_block: header.safe_block, + fee_price: header.fee, + }); + } + Ok(frames) +} + #[cfg(test)] mod tests { use super::super::test_helpers::{ - SENDER_A, SENDER_B, seed_closed_batches, seed_safe_inputs_with_batch_nonces, temp_db, + SENDER_A, SENDER_B, local_batch_payload, seed_closed_batches, + seed_safe_inputs_with_batch_nonces, temp_db, }; use crate::storage::{SafeInputRange, Storage, StoredSafeInput}; use alloy_primitives::Address; - use sequencer_core::batch::{Batch, Frame as BatchFrame}; + use sequencer_core::batch::Batch; use sequencer_core::protocol::ProtocolTiming; #[test] @@ -331,6 +409,28 @@ mod tests { ); } + #[test] + fn wallet_nonce_watermark_is_monotonic_and_absent_at_genesis() { + let db = temp_db("wallet-nonce-watermark"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + + assert_eq!( + storage.wallet_nonce_watermark().expect("read"), + None, + "genesis: nothing ever broadcast" + ); + + storage.raise_wallet_nonce_watermark(5).expect("raise to 5"); + assert_eq!(storage.wallet_nonce_watermark().expect("read"), Some(5)); + + // Monotonic: a lower raise never lowers the watermark. + storage.raise_wallet_nonce_watermark(3).expect("raise to 3"); + assert_eq!(storage.wallet_nonce_watermark().expect("read"), Some(5)); + + storage.raise_wallet_nonce_watermark(9).expect("raise to 9"); + assert_eq!(storage.wallet_nonce_watermark().expect("read"), Some(9)); + } + #[test] fn submitter_frontier_returns_zero_when_no_batches_were_accepted() { let db = temp_db("submitter-frontier-empty"); @@ -343,10 +443,41 @@ mod tests { assert_eq!(frontier.accepted_next_nonce, 0); } + #[test] + fn submitter_frontier_uses_anchor_when_no_batches_accepted() { + // A cockroach-recovered deployment boots with an empty + // safe_accepted_batches and a non-zero batch-tree anchor (N'). The + // submitter's start frontier must be N', not 0 — otherwise it would + // re-submit its first post-recovery batch (nonce N') on every tick + // until the first accepted row lands. Regression test for the + // frontier_nonce/anchor asymmetry (frontier_nonce defaulted to 0 while + // populate_safe_accepted_batches seeds `expected` from the anchor). + let db = temp_db("submitter-frontier-anchor"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); + storage + .append_safe_inputs(0, &[], SENDER_A, &default_test_protocol()) + .expect("record observed safe head"); + let frontier = storage.submitter_frontier().expect("submitter frontier"); + assert_eq!(frontier.accepted_next_nonce, 7); + } + #[test] fn submitter_frontier_tracks_accepted_prefix() { let db = temp_db("submitter-frontier-prefix"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + // Local closed batches 0 and 1 — their landings carry the real wire + // bytes (content-identity check); 3..5 stay synthetic, which is fine + // because the nonce gap means the fold never accepts them. + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close 0"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close 1"); // seed_safe_inputs_with_batch_nonces already calls append_safe_inputs, // which auto-populates safe_accepted_batches. seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0, 1, 3, 4, 5]); @@ -387,19 +518,13 @@ mod tests { .expect("close batch 1"); let protocol = default_test_protocol(); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1135, &[StoredSafeInput { sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&Batch { - nonce: 0, - frames: vec![BatchFrame { - user_ops: vec![], - safe_block: 10, - fee_price: 0, - }], - }), + payload: landed, block_number: 20, }], SENDER_A, @@ -433,19 +558,13 @@ mod tests { .expect("close batch 1"); let protocol = default_test_protocol(); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1200, &[StoredSafeInput { sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&Batch { - nonce: 0, - frames: vec![BatchFrame { - user_ops: vec![], - safe_block: 100, - fee_price: 0, - }], - }), + payload: landed, block_number: 200, }], SENDER_A, @@ -531,19 +650,13 @@ mod tests { .expect("close batch 0; batch 1 is Tip"); let protocol = default_test_protocol(); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1200, &[StoredSafeInput { sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&Batch { - nonce: 0, - frames: vec![BatchFrame { - user_ops: vec![], - safe_block: 10, - fee_price: 0, - }], - }), + payload: landed, block_number: 20, }], SENDER_A, @@ -586,7 +699,14 @@ mod tests { let protocol = default_test_protocol(); let old_safe_timestamp = 1_000_u64; storage - .append_safe_inputs_with_timestamp(1200, old_safe_timestamp, &[], SENDER_A, &protocol) + .append_safe_inputs_with_timestamp( + 1200, + old_safe_timestamp, + &[], + SENDER_A, + &protocol, + crate::storage::FrontierMode::Populate, + ) .expect("advance safe head with stale L1 timestamp"); let now_ms = @@ -619,6 +739,14 @@ mod tests { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let protocol = default_test_protocol(); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + for nonce in 0..4 { + storage + .close_frame_and_batch(&mut head, 10) + .unwrap_or_else(|e| panic!("close {nonce}: {e}")); + } seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0, 1]); // Mixed-sender wave: the SENDER_B row must be ignored, SENDER_A rows @@ -634,18 +762,12 @@ mod tests { }, StoredSafeInput { sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 2, - frames: Vec::new(), - }), + payload: local_batch_payload(&mut storage, 2), block_number: 11, }, StoredSafeInput { sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 3, - frames: Vec::new(), - }), + payload: local_batch_payload(&mut storage, 3), block_number: 11, }, ]; @@ -672,33 +794,27 @@ mod tests { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let protocol = default_test_protocol(); - // Seed a non-stale batch with nonce 0 (safe_block=100, block_number=200, max_wait=1200 → not stale) - let non_stale_payload = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 0, - frames: vec![sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block: 100, - fee_price: 0, - }], - }); - // Seed a stale batch with nonce 1 (safe_block=100, block_number=2000, max_wait=1200 → stale) - let stale_payload = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 1, - frames: vec![sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block: 100, - fee_price: 0, - }], - }); - // Seed a non-stale batch with nonce 1 (safe_block=1900, block_number=2000 → not stale) - let non_stale_payload_2 = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 1, - frames: vec![sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block: 1900, - fee_price: 0, - }], - }); + // Local batch 0 (first frame safe_block=100) and batch 1 (first frame + // safe_block=1900). Their landings carry the real wire bytes so the + // content-identity check accepts them. + let mut head = storage + .initialize_open_state(100, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 1900) + .expect("close 0"); + storage + .close_frame_and_batch(&mut head, 1900) + .expect("close 1"); + + // Non-stale landing of batch 0 (block 200 - safe_block 100 < 1200). + let non_stale_payload = local_batch_payload(&mut storage, 0); + // Stale copy at nonce 1 (safe_block=100, block 2000 → age >= 1200): + // a scheduler no-op, so its content is never compared — synthetic + // bytes are deliberate here. + let stale_payload = super::super::test_helpers::make_stale_batch_payload(1, 100); + // Non-stale landing of batch 1 (block 2000 - safe_block 1900 < 1200). + let non_stale_payload_2 = local_batch_payload(&mut storage, 1); let inputs = vec![ StoredSafeInput { @@ -726,12 +842,109 @@ mod tests { } #[test] - fn frontier_accepts_future_safe_block_batch_by_design() { - // The scheduler rejects batches where frame safe_block > inclusion_block, - // but the sequencer trusts its own output and does not re-validate these - // invariants during recovery. This test documents the intentional design - // choice: populate_safe_accepted_batches accepts such batches because - // the sequencer would never produce them. + fn seal_stamps_payload_hash_of_the_submitter_encode_path() { + // Hash-at-seal (review R2): the hash stamped on the sealed row must + // be the keccak256 of exactly the bytes the submitter will broadcast + // (`pending_batches`'s encoding) — same code path, by construction. + let db = temp_db("seal-stamps-payload-hash"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close batch 0"); + + let stamped: Vec = storage + .conn + .query_row( + "SELECT payload_hash FROM batches WHERE batch_index = 0", + [], + |row| row.get(0), + ) + .expect("sealed batch must carry a payload hash"); + let wire = local_batch_payload(&mut storage, 0); + assert_eq!( + stamped.as_slice(), + alloy_primitives::keccak256(&wire).as_slice(), + "seal-time hash must match the submitter's wire bytes" + ); + } + + #[test] + fn accepted_landing_with_mismatched_content_freezes_frontier() { + // The F1-zombie / F3-re-seal shape: a landing at the expected nonce + // whose bytes differ from the batch we sealed. The check records a + // 'mismatch' marker atomically with the sync and freezes the + // frontier; later syncs stay frozen. + let db = temp_db("mismatch-divergence"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let protocol = default_test_protocol(); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close batch 0"); + + // Same nonce + same first-frame safe block (fresh by inclusion), + // different content (synthetic fee 0 vs the sealed fee). + let imposter = super::super::test_helpers::make_stale_batch_payload(0, 10); + storage + .append_safe_inputs( + 20, + &[StoredSafeInput { + sender: SENDER_A, + payload: imposter, + block_number: 20, + }], + SENDER_A, + &protocol, + ) + .expect("sync commits; the marker rides the same transaction"); + + let kind: String = storage + .conn + .query_row( + "SELECT kind FROM canonical_divergence WHERE singleton_id = 0", + [], + |row| row.get(0), + ) + .expect("marker persisted"); + assert_eq!(kind, "mismatch"); + let frontier = storage.submitter_frontier().expect("frontier"); + assert_eq!(frontier.accepted_next_nonce, 0, "frontier frozen"); + + // A later sync — even one carrying the *correct* bytes — must not + // thaw the frontier: the marker is terminal until cockroach recovery. + let genuine = local_batch_payload(&mut storage, 0); + storage + .append_safe_inputs( + 21, + &[StoredSafeInput { + sender: SENDER_A, + payload: genuine, + block_number: 21, + }], + SENDER_A, + &protocol, + ) + .expect("append after marker"); + let frontier = storage.submitter_frontier().expect("frontier"); + assert_eq!(frontier.accepted_next_nonce, 0, "frontier stays frozen"); + } + + #[test] + fn sim_accepted_foreign_batch_freezes_frontier_with_divergence_marker() { + // `scheduler_accepts` deliberately omits the two structural + // rejections (future safe_block, non-monotonic frames) under + // self-trust — the sequencer never produces them. Before the + // content-identity check (review R2), such a foreign batch at the + // expected nonce would be sim-accepted and silently desync the + // frontier forever. With the check, it fails the local-batch lookup + // (kind = foreign), the poison marker persists atomically with the + // sync, the frontier freezes, and `check_danger` reports + // `CanonicalDivergence` ahead of every other arm. let db = temp_db("frontier-future-safe-block"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); @@ -743,21 +956,6 @@ mod tests { fee_price: 0, }], }); - let non_monotonic_payload = ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 1, - frames: vec![ - sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block: 200, - fee_price: 0, - }, - sequencer_core::batch::Frame { - user_ops: Vec::new(), - safe_block: 100, - fee_price: 0, - }, - ], - }); let batch_submitter = Address::repeat_byte(0xCC); let protocol = ProtocolTiming { @@ -766,26 +964,146 @@ mod tests { l1_read_stale_after_blocks: 900, seconds_per_block: 12, }; - let inputs = vec![ - StoredSafeInput { - sender: batch_submitter, - payload: future_safe_block_payload, - block_number: 100, - }, - StoredSafeInput { - sender: batch_submitter, - payload: non_monotonic_payload, - block_number: 200, - }, - ]; + let inputs = vec![StoredSafeInput { + sender: batch_submitter, + payload: future_safe_block_payload, + block_number: 100, + }]; storage .append_safe_inputs(200, inputs.as_slice(), batch_submitter, &protocol) - .expect("append"); + .expect("append commits; the marker rides the same transaction"); let frontier = storage.submitter_frontier().expect("submitter frontier"); assert_eq!( - frontier.accepted_next_nonce, 2, - "both batches should be in accepted frontier" + frontier.accepted_next_nonce, 0, + "the frontier must freeze before the diverged landing" + ); + let status = storage + .check_danger(&protocol, unix_now_ms()) + .expect("check_danger"); + assert_eq!( + status, + crate::storage::DangerStatus::CanonicalDivergence(0), + "the marker outranks every other danger arm" + ); + } + + #[test] + fn populate_accepts_post_recovery_batch_at_anchor() { + // A cockroach-recovered deployment is anchored at N' > 0; its first + // post-recovery batch lands at nonce N'. populate must seed `expected` + // from the anchor and accept it (frontier → N'+1), content-identity + // passing against the genuine local batch. Every other populate test + // uses anchor=0, so the N' > 0 acceptance path had no coverage. + let db = temp_db("populate-at-anchor"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let protocol = default_test_protocol(); + let submitter = SENDER_A; + + // Root the tree at N' = 7, then build + seal one local batch — it carries + // nonce 7 (compute_next_nonce(None) reads the anchor). + storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close the nonce-7 batch"); + assert_eq!( + storage.batch_nonce(0).expect("nonce"), + 7, + "the root tip carries the anchor nonce N'" + ); + + // Its genuine wire bytes (looked up by nonce N'=7) land from the + // submitter at a fresh inclusion block. + let genuine = local_batch_payload(&mut storage, 7); + storage + .append_safe_inputs( + 20, + &[StoredSafeInput { + sender: submitter, + payload: genuine, + block_number: 20, + }], + submitter, + &protocol, + ) + .expect("sync the N' landing"); + + assert_eq!( + storage + .submitter_frontier() + .expect("frontier") + .accepted_next_nonce, + 8, + "the N'=7 batch is accepted; the frontier advances to 8" + ); + let divergence: i64 = storage + .conn + .query_row("SELECT COUNT(*) FROM canonical_divergence", [], |r| { + r.get(0) + }) + .expect("count"); + assert_eq!( + divergence, 0, + "a genuine N' landing must not flag divergence" + ); + } + + #[test] + fn populate_skips_below_anchor_collapsed_history_without_divergence() { + // Below the anchor the local tree has no batches (that history was folded + // into S'). Old-generation L1 landings at nonces < N' are trusted + // collapsed history: skipped by nonce-mismatch BEFORE the content-identity + // check, so they must NOT be flagged Foreign (which would freeze the + // frontier and force a false recovery). Pins the anchor-aware-frontier + // (I15) skip arm — the inverse of the foreign-batch freeze test above. + let db = temp_db("populate-below-anchor"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let protocol = default_test_protocol(); + let submitter = SENDER_A; + storage.set_batch_tree_anchor(7).expect("anchor at N'=7"); + + // Old-generation batches at nonces 0, 1, 2 (< anchor) land from the submitter. + let mk = |nonce: u64| { + ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { + nonce, + frames: vec![sequencer_core::batch::Frame { + user_ops: Vec::new(), + safe_block: 10, + fee_price: 0, + }], + }) + }; + let landings: Vec = (0..3) + .map(|n| StoredSafeInput { + sender: submitter, + payload: mk(n), + block_number: 20 + n, + }) + .collect(); + storage + .append_safe_inputs(30, &landings, submitter, &protocol) + .expect("sync below-anchor landings"); + + let divergence: i64 = storage + .conn + .query_row("SELECT COUNT(*) FROM canonical_divergence", [], |r| { + r.get(0) + }) + .expect("count"); + assert_eq!( + divergence, 0, + "below-anchor history must be skipped by nonce-mismatch, not flagged Foreign" + ); + assert_eq!( + storage + .submitter_frontier() + .expect("frontier") + .accepted_next_nonce, + 7, + "nothing is accepted; the frontier stays at the anchor N'" ); } @@ -875,18 +1193,19 @@ mod tests { .expect("init"); storage.close_frame_and_batch(&mut head, 10).expect("close"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[ StoredSafeInput { sender: SENDER_A, - payload: super::super::test_helpers::make_stale_batch_payload(0, 10), + payload: landed.clone(), block_number: 20, }, StoredSafeInput { sender: SENDER_A, - payload: super::super::test_helpers::make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }, ], @@ -963,12 +1282,13 @@ mod tests { "out of order must stall frontier" ); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 21, &[StoredSafeInput { sender: SENDER_A, - payload: super::super::test_helpers::make_stale_batch_payload(0, 10), + payload: landed, block_number: 21, }], SENDER_A, diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index dc90163..20a5da4 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -19,15 +19,27 @@ -- identity; reused across recovery cascades (new Tip forks from last valid -- ancestor, inheriting nonce via the +1 rule). -- --------------------------------------------------------------------------- +-- `sealed_at_ms` / `invalidated_at_ms` are observability stamps from the +-- wall clock — write-once (triggers below), but deliberately NOT +-- cross-checked against `created_at_ms`: wall-clock monotonicity is an +-- environmental assumption, not an invariant (NTP steps, VM resume), and a +-- CHECK on it wedged batch close and the recovery cascade in a clock +-- regression (review F8). No production code reads these as values; every +-- reader is an IS NULL / IS NOT NULL predicate. +-- `payload_hash` is the keccak256 of the batch's SSZ wire bytes, stamped at +-- seal time by the same encode path the submitter uses (review R2, +-- hash-at-seal). It is what the content-identity check compares an accepted +-- L1 landing against — and because it is computed by the code that sealed +-- the batch, it survives wire-format upgrades. NULL only while the batch is +-- the open Tip (and on recovery sentinels, which carry no payload). CREATE TABLE IF NOT EXISTS batches ( batch_index INTEGER PRIMARY KEY, parent_batch_index INTEGER REFERENCES batches(batch_index), -- NULL only for genesis nonce INTEGER NOT NULL CHECK (nonce >= 0), created_at_ms INTEGER NOT NULL, - sealed_at_ms INTEGER - CHECK (sealed_at_ms IS NULL OR sealed_at_ms >= created_at_ms), - invalidated_at_ms INTEGER - CHECK (invalidated_at_ms IS NULL OR invalidated_at_ms >= created_at_ms) + sealed_at_ms INTEGER CHECK (sealed_at_ms IS NULL OR sealed_at_ms >= 0), + invalidated_at_ms INTEGER CHECK (invalidated_at_ms IS NULL OR invalidated_at_ms >= 0), + payload_hash BLOB CHECK (payload_hash IS NULL OR length(payload_hash) = 32) ); -- "At most one valid Tip" — structural via partial unique index. The predicate @@ -57,6 +69,26 @@ CREATE VIEW IF NOT EXISTS valid_closed_batches AS CREATE VIEW IF NOT EXISTS valid_open_batch AS SELECT * FROM valid_batches WHERE sealed_at_ms IS NULL; +-- Batch-tree anchor: the nonce the single (valid) parentless root carries. +-- +-- A genesis deployment is anchored at 0; a cockroach-recovered one is anchored +-- at N' (the post-checkpoint resume nonce), so `run`'s first tip roots at N' +-- without replaying history — there is no separate "sentinel" batch row, the +-- root tip *is* the anchor. The value generalizes the rule the tree already +-- has ("a parentless root carries nonce 0") from a hard-coded 0 to this +-- singleton; it is read by `trg_enforce_nonce_contiguity` (below) and by +-- `compute_next_nonce(parent = None)`. +-- +-- Default 0, so a normal deployment is byte-identical to before this table +-- existed. Written exactly once — by `setup` (recovery sets N' before the +-- `setup_complete` marker); `trg_batch_tree_anchor_write_once` freezes it +-- thereafter, since re-anchoring a live deployment would strand its spine. +CREATE TABLE IF NOT EXISTS batch_tree_anchor ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + nonce INTEGER NOT NULL CHECK (nonce >= 0) +); +INSERT OR IGNORE INTO batch_tree_anchor(singleton_id, nonce) VALUES (0, 0); + -- ── Triggers ─────────────────────────────────────────────────────────────── -- -- These enforce invariants the writer could otherwise violate with a bug. @@ -65,14 +97,29 @@ CREATE VIEW IF NOT EXISTS valid_open_batch AS -- transition sequence — triggers just ensure the DB never reaches an -- inconsistent state if the writer misbehaves. --- Nonce contiguity: `nonce = parent.nonce + 1`, or 0 for genesis. +-- Nonce contiguity: `nonce = parent.nonce + 1`, or the batch-tree anchor nonce +-- (0 for a genesis deployment, N' for a recovered one) for the parentless root. CREATE TRIGGER IF NOT EXISTS trg_enforce_nonce_contiguity AFTER INSERT ON batches FOR EACH ROW BEGIN SELECT CASE - WHEN NEW.parent_batch_index IS NULL AND NEW.nonce != 0 - THEN RAISE(ABORT, 'genesis batch must have nonce 0') + -- A parentless root must carry the deployment's anchor nonce. This is + -- an *exact* match — tighter than the old "must be 0": a buggy root at + -- any other nonce still ABORTs, including a re-root at 0 on a recovered + -- (anchor = N') deployment. + WHEN NEW.parent_batch_index IS NULL + AND NEW.nonce != (SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0) + THEN RAISE(ABORT, 'parentless root must carry the batch-tree anchor nonce') + -- At most one *valid* parentless root. A fully-torn cascade invalidates + -- the old root, then re-roots parentless at the anchor (the documented + -- `open_fresh_tip_in_tx` parent=None path) — leaving exactly one valid + -- root again, with the invalidated old root(s) coexisting. (Counts the + -- just-inserted row, hence `> 1`.) + WHEN NEW.parent_batch_index IS NULL + AND (SELECT COUNT(*) FROM batches + WHERE parent_batch_index IS NULL AND invalidated_at_ms IS NULL) > 1 + THEN RAISE(ABORT, 'at most one valid parentless root per deployment') WHEN NEW.parent_batch_index IS NOT NULL AND NEW.nonce != (SELECT nonce + 1 FROM batches WHERE batch_index = NEW.parent_batch_index) THEN RAISE(ABORT, 'batch nonce must equal parent.nonce + 1') @@ -97,6 +144,16 @@ BEGIN SELECT RAISE(ABORT, 'invalidated_at_ms is write-once'); END; +-- Write-once: payload_hash transitions only NULL → non-NULL (stamped in the +-- same UPDATE that seals the batch). +CREATE TRIGGER IF NOT EXISTS trg_payload_hash_write_once +BEFORE UPDATE OF payload_hash ON batches +FOR EACH ROW +WHEN OLD.payload_hash IS NOT NULL +BEGIN + SELECT RAISE(ABORT, 'payload_hash is write-once'); +END; + -- parent_batch_index is immutable after insert. CREATE TRIGGER IF NOT EXISTS trg_parent_batch_index_immutable BEFORE UPDATE OF parent_batch_index ON batches @@ -263,7 +320,7 @@ WHERE batch_index NOT IN (SELECT batch_index FROM batches WHERE invalidated_at_m -- Unlike a raw log of all safe submissions, this only contains the accepted -- prefix: batches whose nonce matched the expected sequence and were not stale. -- Maintained atomically by Storage::append_safe_inputs (via --- populate_safe_accepted_batches_inner), which simulates the scheduler's +-- populate_safe_accepted_batches), which simulates the scheduler's -- acceptance logic over new safe_inputs rows. CREATE TABLE IF NOT EXISTS safe_accepted_batches ( safe_input_index INTEGER PRIMARY KEY REFERENCES safe_inputs(safe_input_index), @@ -283,6 +340,38 @@ CREATE TABLE IF NOT EXISTS l1_safe_head ( synced_at_ms INTEGER NOT NULL CHECK (synced_at_ms >= 0) ); +-- Highest wallet nonce ever broadcast by this deployment's batch-submitter +-- key (review R1a — the durable realization of the TLA+ spec's +-- `walletNonce`). Write-before-broadcast: any component about to send a tx +-- at wallet nonce n first commits watermark = max(watermark, n) — power-loss +-- durable under synchronous=FULL — then sends. Uniform for batch txs and +-- flush no-ops alike, so the flush's slot coverage never depends on the +-- local node's volatile mempool memory (the F1 zombie). Absent row = +-- nothing ever broadcast. Never reset, never lowered. +CREATE TABLE IF NOT EXISTS wallet_nonce_watermark ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + watermark INTEGER NOT NULL CHECK (watermark >= 0) +); + +-- Canonical-divergence poison marker (review R2). Written by the input +-- reader's acceptance simulation — atomically with the sync that detected +-- it — when a fully-accepted L1 landing fails the content-identity check: +-- either no valid closed local batch exists at the accepted nonce +-- (kind = 'foreign': a zombie or foreign batch from our key) or the landed +-- bytes hash differently from ours (kind = 'mismatch'). Once present, the +-- acceptance frontier freezes, `check_danger` reports `CanonicalDivergence` +-- ahead of every other arm, startup refuses, and the runtime detector exits. +-- The remedy is cockroach recovery (wipe + rebuild from L1), never standard +-- recovery — canonical state contains executed effects with no reliable +-- local source. Keep-first: only the earliest detection is recorded. +CREATE TABLE IF NOT EXISTS canonical_divergence ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + nonce INTEGER NOT NULL CHECK (nonce >= 0), + safe_input_index INTEGER NOT NULL CHECK (safe_input_index >= 0), + kind TEXT NOT NULL CHECK (kind IN ('foreign', 'mismatch')), + detected_at_ms INTEGER NOT NULL CHECK (detected_at_ms >= 0) +); + -- Deployment identity: the persisted DB is only valid for this deployment. -- Allows L1-unreachable startup after first boot, and prevents interpreting -- historical sequencer state under a different app or batch-submitter address. @@ -295,6 +384,32 @@ CREATE TABLE IF NOT EXISTS deployment_identity ( batch_submitter_address BLOB NOT NULL CHECK (length(batch_submitter_address) = 20) ); +-- setup-complete marker. The `setup` subcommand +-- pins deployment identity, does the initial L1 sync, and registers the +-- genesis finalized snapshot; it inserts this singleton row as its LAST +-- write. `run` refuses to boot unless the row is present. Presence is the +-- single linearization point for "setup finished": it distinguishes a clean +-- setup from one that crashed midway (identity pinned and/or genesis +-- snapshot registered, but the marker absent), which every prior setup step +-- is individually idempotent enough to let `setup` re-run and complete. +CREATE TABLE IF NOT EXISTS setup_complete ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + completed_at_ms INTEGER NOT NULL CHECK (completed_at_ms >= 0) +); + +-- The batch-tree anchor is frozen once setup completes. `setup` writes it (0 +-- by default, N' on recovery) before inserting the `setup_complete` marker; +-- after the marker exists, re-anchoring a live deployment would strand the +-- existing batch spine, so any further UPDATE aborts. Defense-in-depth: the +-- `setup --recovery` path is already a strict one-shot on a fresh DB. +CREATE TRIGGER IF NOT EXISTS trg_batch_tree_anchor_write_once +BEFORE UPDATE OF nonce ON batch_tree_anchor +FOR EACH ROW +WHEN EXISTS (SELECT 1 FROM setup_complete WHERE singleton_id = 0) +BEGIN + SELECT RAISE(ABORT, 'batch-tree anchor is frozen after setup completes'); +END; + -- --------------------------------------------------------------------------- -- Batch policy singleton diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 16fd884..20d2f6d 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -60,6 +60,24 @@ pub struct StoredSafeInput { pub block_number: u64, } +/// Whether a sync also maintains the scheduler-accepted gold frontier +/// (`safe_accepted_batches`). +/// +/// Every normal sync uses [`FrontierMode::Populate`]. `setup --recovery` uses +/// [`FrontierMode::DeferUntilAnchorSet`] for its interim syncs: the local batch +/// tree is rebuilt from the checkpoint *after* the sync, so populating the +/// frontier against the empty tree would mark every L1 batch `Foreign` and +/// falsely freeze it. `run`'s first sync populates it once the anchor is `N'`. +/// See `docs/recovery/cockroach.md` and the anchor-aware-frontier note on I15. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrontierMode { + /// Maintain `safe_accepted_batches` as part of the sync (the default). + Populate, + /// Skip frontier maintenance — the tree is rebuilt and the anchor set + /// afterwards, and the next normal sync will populate it correctly. + DeferUntilAnchorSet, +} + /// Half-open range `[start, end)` over `safe_input_index` values. Used to /// describe which safe inputs a frame drained. /// diff --git a/sequencer/src/storage/mutations.rs b/sequencer/src/storage/mutations.rs index 39203f1..88dab7d 100644 --- a/sequencer/src/storage/mutations.rs +++ b/sequencer/src/storage/mutations.rs @@ -7,7 +7,7 @@ //! larger atomic unit. The two consumers today are ingress (batch/frame close //! + re-drain) and recovery (opening a recovery batch after cascade). -use rusqlite::{Result, Transaction, params}; +use rusqlite::{Connection, Result, Transaction, params}; use super::SafeInputRange; use super::convert::{i64_to_u64, u64_to_i64}; @@ -60,7 +60,12 @@ pub(super) fn insert_new_batch( fn compute_next_nonce(tx: &Transaction<'_>, parent_batch_index: Option) -> Result { match parent_batch_index { - None => Ok(0), + // A parentless root carries the deployment's batch-tree anchor nonce: + // 0 for a genesis deployment, N' for a cockroach-recovered one. This + // generalizes the old hard-coded 0; the `batch_tree_anchor` row defaults + // to 0, so genesis and post-cascade re-roots are unchanged. Mirrored by + // `trg_enforce_nonce_contiguity`'s parentless arm. + None => batch_tree_anchor_in(tx), Some(parent_bi) => { let parent_nonce: i64 = tx.query_row( "SELECT nonce FROM batches WHERE batch_index = ?1", @@ -73,11 +78,23 @@ fn compute_next_nonce(tx: &Transaction<'_>, parent_batch_index: Option) -> } /// Mark a batch as sealed (inclusion lane closed it). Write-once per the -/// `trg_sealed_at_ms_write_once` trigger. -pub(super) fn seal_batch(tx: &Transaction<'_>, batch_index: u64, sealed_at_ms: i64) -> Result<()> { +/// `trg_sealed_at_ms_write_once` / `trg_payload_hash_write_once` triggers. +/// The payload hash is stamped in the same UPDATE: a sealed batch always +/// carries the hash the content-identity check compares accepted L1 +/// landings against (review R2, hash-at-seal). +pub(super) fn seal_batch( + tx: &Transaction<'_>, + batch_index: u64, + sealed_at_ms: i64, + payload_hash: &[u8; 32], +) -> Result<()> { let changed = tx.execute( - "UPDATE batches SET sealed_at_ms = ?1 WHERE batch_index = ?2", - params![sealed_at_ms, u64_to_i64(batch_index)], + "UPDATE batches SET sealed_at_ms = ?1, payload_hash = ?2 WHERE batch_index = ?3", + params![ + sealed_at_ms, + payload_hash.as_slice(), + u64_to_i64(batch_index) + ], )?; if changed != 1 { return Err(rusqlite::Error::StatementChangedRows(changed)); @@ -107,6 +124,37 @@ pub(super) fn insert_open_frame( Ok(()) } +/// Set the batch-tree anchor nonce — the nonce the single parentless root +/// carries (0 for genesis, `N'` for a cockroach-recovered deployment). Composes +/// inside the recovery-fill transaction. `trg_batch_tree_anchor_write_once` +/// rejects this once `setup_complete` exists, so it is callable only during +/// setup, before the marker. +pub(super) fn set_batch_tree_anchor_in(tx: &Transaction<'_>, nonce: u64) -> Result<()> { + let changed = tx.execute( + "UPDATE batch_tree_anchor SET nonce = ?1 WHERE singleton_id = 0", + params![u64_to_i64(nonce)], + )?; + if changed != 1 { + return Err(rusqlite::Error::StatementChangedRows(changed)); + } + Ok(()) +} + +/// Read the batch-tree anchor nonce (default 0). Mirrors +/// [`set_batch_tree_anchor_in`]. The single home for the anchor-read query: +/// takes `&Connection` (not `&Transaction`) so both the in-transaction writer +/// ([`compute_next_nonce`]) and the connection-level frontier readers +/// (`frontier_nonce`, `populate_safe_accepted_batches`) share it. A +/// `&Transaction` coerces to `&Connection` at the call site. +pub(super) fn batch_tree_anchor_in(conn: &Connection) -> Result { + let anchor: i64 = conn.query_row( + "SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0", + [], + |row| row.get(0), + )?; + Ok(i64_to_u64(anchor)) +} + /// Insert one `sequenced_l2_txs` row per safe-input index in `range` for the /// given (batch, frame). Used by ingress (frame close) and recovery (re-drain /// after cascade invalidation). diff --git a/sequencer/src/storage/open.rs b/sequencer/src/storage/open.rs index 2e752d6..5e61bda 100644 --- a/sequencer/src/storage/open.rs +++ b/sequencer/src/storage/open.rs @@ -14,10 +14,28 @@ use super::StorageOpenError; const MIGRATION_0001_SCHEMA: &str = include_str!("migrations/0001_schema.sql"); /// SQLite `synchronous` pragma used by every production writer connection. -/// `NORMAL` is appropriate under WAL — fsyncs at checkpoint boundaries, not -/// per-transaction. Tests use the same value; if a future test needs -/// `FULL`/`OFF`, add a `#[cfg(test)]` override. -const SYNCHRONOUS_PRAGMA: &str = "NORMAL"; +/// `FULL` under WAL fsyncs on every commit, so commits survive power loss / +/// OS crash — not just process crash. Load-bearing (review R3/F3): the +/// sequencer externalizes effects on commits (acks `POST /tx` after the +/// chunk commit; the submitter broadcasts sealed batches), and a rewound +/// commit after externalization is silent divergence — e.g. a re-sealed +/// batch at the same nonce with different content than the one the +/// scheduler executed. The dump side already pays the same cost +/// (`create_dump` fsyncs); this closes the DB half. Also a precondition +/// for the wallet-nonce watermark's write-before-broadcast guarantee +/// (review R1a). And it is what makes the `setup_complete` marker a valid +/// linearization point: the marker is committed in its own transaction +/// after the genesis-snapshot row's transaction, so "marker durable ⇒ +/// snapshot row durable ⇒ dump dir durable" only holds because FULL fsyncs +/// every commit — under NORMAL the marker's WAL frame could survive while +/// the snapshot row's frames are lost, and `run` would boot a half-set-up +/// DB. Benchmarked at the flip: round-trip/ack deltas were noise-level on +/// NVMe (see review ledger WP1). +/// +/// Do not relax to NORMAL without revisiting all three (R3/F3 externalized +/// commits, R1a watermark, the setup-marker linearization in +/// `runtime/setup.rs` + `storage/migrations/0001_schema.sql`). +const SYNCHRONOUS_PRAGMA: &str = "FULL"; /// Sequencer storage backed by a single SQLite database. /// @@ -114,7 +132,8 @@ impl Storage { } } -/// Open a read-write connection with WAL + `NORMAL` sync + 5s busy timeout. +/// Open a read-write connection with WAL + `FULL` sync (`SYNCHRONOUS_PRAGMA`) + +/// 5s busy timeout. fn open_writer_connection(path: &str) -> Result { let conn = Connection::open(path)?; conn.pragma_update(None, "foreign_keys", "ON")?; diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs index a4576ae..c8ebd9a 100644 --- a/sequencer/src/storage/recovery.rs +++ b/sequencer/src/storage/recovery.rs @@ -31,8 +31,8 @@ use super::ingress::open_fresh_tip_in_tx; use super::queries::{ current_safe_block_required, current_safe_block_timestamp, last_safe_progress_ms, }; -use super::safe_accepted_batches::frontier_nonce; -use super::snapshot_dumps::clear_pending_dumps_in; +use super::safe_accepted_batches::{canonical_divergence_in, frontier_nonce}; +use super::snapshot_dumps::{batch_nonce_in, clear_pending_dumps_from_nonce_in}; /// Outcome of a danger-zone check. /// @@ -59,6 +59,14 @@ use super::snapshot_dumps::clear_pending_dumps_in; pub enum DangerStatus { /// No danger detected — none of the checks tripped. Safe, + /// A fully-accepted L1 landing failed the content-identity check + /// (review R2): canonical state contains executed effects with no + /// reliable local source. Carries the diverged batch nonce. Ranked + /// ahead of every other arm so the respawn loop can never route a + /// diverged node into `Proceed`/`FlushAndCascade`. The remedy is + /// cockroach recovery (wipe + rebuild from L1), never standard + /// recovery. + CanonicalDivergence(u64), /// L1 safe-head timestamp is too old or unknown. Recovery cannot reason /// from the local L1 view, so startup must refuse. L1ViewStale, @@ -78,6 +86,35 @@ pub enum DangerStatus { EstimatedBatchInDanger(u64), } +impl DangerStatus { + /// Stable label for logs/metrics. An inherent method (not a free + /// projection) so a new variant must add its label right here. + pub(crate) fn label(self) -> &'static str { + match self { + DangerStatus::Safe => "safe", + DangerStatus::CanonicalDivergence(_) => "canonical_divergence", + DangerStatus::L1ViewStale => "l1_view_stale", + DangerStatus::ClosedBatchInDanger(_) => "closed_batch_in_danger", + DangerStatus::TipInDanger(_) => "tip_in_danger", + DangerStatus::EstimatedBatchInDanger(_) => "estimated_batch_in_danger", + } + } + + /// The batch nonce a danger arm points at, if any (log context). The + /// `CanonicalDivergence` nonce is deliberately not reported here — it is a + /// diverged-state nonce, not a batch in the danger pipeline. + pub(crate) fn batch_index(self) -> Option { + match self { + DangerStatus::ClosedBatchInDanger(batch_index) + | DangerStatus::TipInDanger(batch_index) + | DangerStatus::EstimatedBatchInDanger(batch_index) => Some(batch_index), + DangerStatus::Safe + | DangerStatus::L1ViewStale + | DangerStatus::CanonicalDivergence(_) => None, + } + } +} + impl Storage { /// Unified danger-zone detection. /// @@ -124,6 +161,14 @@ impl Storage { /// callers pass the current Unix-ms clock. pub fn check_danger(&mut self, protocol: &ProtocolTiming, now_ms: u64) -> Result { self.read(|tx| { + // The divergence marker outranks everything — including the + // L1-staleness gate: it records an already-confirmed fact about + // canonical state, not a view-dependent estimate, and no amount + // of L1 freshening or flushing repairs it (review R2). + if let Some((nonce, _)) = canonical_divergence_in(tx)? { + return Ok(DangerStatus::CanonicalDivergence(nonce)); + } + if protocol.l1_view_is_stale(current_safe_block_timestamp(tx)?, now_ms) { return Ok(DangerStatus::L1ViewStale); } @@ -241,7 +286,8 @@ impl Storage { /// Cascade the open Tip if its first frame has aged past /// `danger_threshold`. Called from the `RecoverTip` startup path (no flush - /// happened), and defensively from `Proceed`. + /// happened). The `Proceed` path performs no DB writes and does not call + /// this. /// /// # Why a threshold here, but no closed-frontier check /// @@ -253,9 +299,8 @@ impl Storage { /// The Tip is different: it has no L1 footprint at all (no `w_nonce`, /// no `safe_input`), so there's no L1 outcome to wait on. Once its /// first frame has aged into the danger zone, the rule "everything - /// past gold is bad once we're committed to recovery" applies. In the - /// `RecoverTip` path startup is already committed; in `Proceed`, this - /// branch is defensive and should normally be a no-op. + /// past gold is bad once we're committed to recovery" applies, and in + /// the `RecoverTip` path startup is already committed. /// /// # Threshold = danger_threshold, not MAX_WAIT /// @@ -291,42 +336,50 @@ fn recover_post_flush_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Resu // in the danger zone — see `recover_post_flush` doc on Tip handling. None => find_tip_batch_in_danger(tx, danger_threshold)?, }; - let invalidated = match pivot { - Some(batch_index) => cascade_invalidate_from(tx, batch_index)?, - None => Vec::new(), - }; - if !invalidated.is_empty() { - // Pending snapshots correspond to batches that haven't been - // observed landing on L1 yet. Once those batches are - // cascade-invalidated, the snapshots represent states the - // canonical replay will never reach — leaving them would - // poison catch-up after restart. Finalized is untouched - // because its bytes are for an L1-confirmed batch. - clear_pending_dumps_in(tx)?; - } - if !invalidated.is_empty() || !has_valid_open_batch(tx)? { - // Reopen the Tip the cascade just invalidated (or one a torn crash - // left missing), atomically with the cascade. Same mechanism the - // runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. - open_fresh_tip_in_tx(tx)?; - } - Ok(invalidated) + cascade_and_reopen(tx, pivot) } /// See [`Storage::recover_aging_tip`] for the design rationale. fn recover_aging_tip_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Result> { - let invalidated = match find_tip_batch_in_danger(tx, danger_threshold)? { - Some(batch_index) => cascade_invalidate_from(tx, batch_index)?, + let pivot = find_tip_batch_in_danger(tx, danger_threshold)?; + cascade_and_reopen(tx, pivot) +} + +/// Shared tail of both recovery paths — the pivot selection above is the +/// only thing that varies. In the caller's transaction: +/// +/// 1. **Cascade** from `pivot` (no-op when `None`): invalidate it and every +/// successor, including the open Tip. +/// 2. **Clear doomed pending snapshots, scoped to the cascade**: delete +/// pending rows with `nonce >= pivot.nonce` — exactly the cascaded +/// batches' pendings, states the canonical replay will never reach. +/// Gold-but-unpromoted pendings (batches that landed while the process +/// was down) carry lower nonces and *survive*: catch-up resumes from a +/// fresher checkpoint, and the rows are cleaned up by the next +/// promotion's `DELETE <= max_nonce`. Scoping is load-bearing (review +/// F9): a blanket clear would arm a promote-wedge crash-loop whenever a +/// *valid in-flight* closed batch existed at clear time — its pending +/// row would be deleted while the batch stayed valid, and the lane's +/// later promotion of its landing would hit the deleted row with no +/// danger arm ever firing to heal it. With the scope, any nonce the +/// lane can later observe as accepted either has its pending row intact +/// or belongs to a post-recovery batch with a fresh row. In the +/// `RecoverTip` path the scope deletes nothing — the Tip never has a +/// pending row. Finalized is untouched (L1-confirmed bytes). +/// 3. **Reopen the Tip** the cascade just invalidated (or one a torn crash +/// left missing), atomically with the cascade. Same mechanism the +/// runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. +fn cascade_and_reopen(tx: &Transaction<'_>, pivot: Option) -> Result> { + let invalidated = match pivot { + Some(batch_index) => { + let pivot_nonce = batch_nonce_in(tx, batch_index)?; + let invalidated = cascade_invalidate_from(tx, batch_index)?; + clear_pending_dumps_from_nonce_in(tx, pivot_nonce)?; + invalidated + } None => Vec::new(), }; - if !invalidated.is_empty() { - // See `recover_post_flush_inner` for why we clear pending here. - clear_pending_dumps_in(tx)?; - } if !invalidated.is_empty() || !has_valid_open_batch(tx)? { - // Reopen the Tip the cascade just invalidated (or one a torn crash - // left missing), atomically with the cascade. Same mechanism the - // runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. open_fresh_tip_in_tx(tx)?; } Ok(invalidated) @@ -365,10 +418,11 @@ fn first_non_gold_closed_batch(conn: &Connection) -> Result> { /// [`Storage::check_danger`]'s wall-clock-adjusted arm, where the dispatch /// is the same (`Refuse`) regardless of which one fired. /// -/// Closed-frontier wins ties: if a closed batch is in danger, the Tip is -/// older still (sequencer opens new batches at non-decreasing `safe_block`), -/// and cascading from the closed batch covers the Tip via -/// `batch_index >= N`. +/// Closed-frontier wins: frame `safe_block`s are non-decreasing along the +/// spine, so the closed frontier is at least as *old* as the Tip — whenever +/// the Tip is in danger, the closed frontier is too, and cascading from the +/// closed batch covers the Tip via `batch_index >= N`. (This ordering is +/// load-bearing for the pending-snapshot clear — see `docs/invariants.md`.) /// /// Reads `safe_accepted_batches`, which is maintained atomically with each /// [`Storage::append_safe_inputs`] call. diff --git a/sequencer/src/storage/recovery_tests.rs b/sequencer/src/storage/recovery_tests.rs index 2fc07a7..956cd5f 100644 --- a/sequencer/src/storage/recovery_tests.rs +++ b/sequencer/src/storage/recovery_tests.rs @@ -1,6 +1,6 @@ use super::super::test_helpers::{ - SENDER_A, all_ordered_l2_txs, default_protocol_timing, make_stale_batch_payload, - seed_closed_batches, seed_safe_inputs_with_batch_nonces, temp_db, + SENDER_A, all_ordered_l2_txs, default_protocol_timing, local_batch_payload, + make_stale_batch_payload, seed_closed_batches, seed_safe_inputs_with_batch_nonces, temp_db, }; use super::{find_closed_frontier_batch_in_danger, find_first_batch_in_danger}; use crate::storage::{SafeInputRange, Storage, StoredSafeInput}; @@ -284,12 +284,13 @@ mod recover_post_flush { storage .close_frame_and_batch(&mut head, 1300) .expect("close recovery batch"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1310, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 1300), + payload: landed, block_number: 1310, }], SENDER_A, @@ -392,12 +393,13 @@ mod recover_post_flush { // is well below MAX_WAIT, so populate accepts it as gold. // Then advance safe head to 1100: batch 0 age (gold) is irrelevant, // but Tip's age = 1100 - 10 = 1090 > danger_threshold (let's pass 1000). + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 200, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 200, }], SENDER_A, @@ -446,12 +448,13 @@ mod recover_post_flush { .expect("close batch 0"); let batch_submitter = Address::repeat_byte(0xAA); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 200, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 200, }], SENDER_A, @@ -1093,12 +1096,13 @@ mod check_danger_zone { .close_frame_and_batch(&mut head, 100) .expect("close batch 0"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -1221,12 +1225,13 @@ mod check_any_unresolved { .close_frame_and_batch(&mut head, 100) .expect("close batch 1"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -1258,12 +1263,13 @@ mod check_any_unresolved { .close_frame_and_batch(&mut head, 100) .expect("close batch 1"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -1281,6 +1287,67 @@ mod check_any_unresolved { "should not trigger below threshold; got batch_index={result:?}" ); } + + #[test] + fn find_first_batch_in_danger_prefers_closed_frontier_over_aged_tip() { + // When BOTH the closed frontier batch and the open Tip are aged past the + // threshold, find_first_batch_in_danger must return the CLOSED frontier: + // cascading from it covers the Tip (batch_index >= pivot), and the scoped + // pending-snapshot clear keys on the pivot's nonce (F9). Both existing + // find_*_in_danger tests use open-batch-only scenarios; the closed-over-Tip + // preference (the helper's whole point) was unasserted. + let db = temp_db("danger-prefers-closed-frontier"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + + // batch 0 (frame sb 10) → made gold below; batch 1 (frame sb 50) → the + // closed frontier; batch 2 (Tip, frame sb 50) → open at the same safe + // block (the lane rotated without a safe-block advance). + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 50) + .expect("seal batch 0, open batch 1 @ sb 50"); + storage + .close_frame_and_batch(&mut head, 50) + .expect("seal batch 1, open the Tip @ sb 50"); + + // Make batch 0 gold: land its genuine bytes (accepted) → frontier → nonce 1. + let landed = local_batch_payload(&mut storage, 0); + storage + .append_safe_inputs( + 20, + &[StoredSafeInput { + sender: SENDER_A, + payload: landed, + block_number: 20, + }], + SENDER_A, + &default_protocol_timing(), + ) + .expect("land batch 0"); + assert_eq!( + storage + .submitter_frontier() + .expect("frontier") + .accepted_next_nonce, + 1, + "batch 0 is gold; the closed frontier is now batch 1 (nonce 1)" + ); + + // Advance the safe head so both batch 1 and the Tip (both sb 50) are aged: + // 1200 - 50 = 1150 >= threshold 1125. + storage + .append_safe_inputs(1200, &[], SENDER_A, &default_protocol_timing()) + .expect("advance safe head"); + + let result = find_first_batch_in_danger(&storage.conn, 1125).expect("find first in danger"); + assert_eq!( + result, + Some(1), + "must return the closed frontier batch (1), not the aged Tip (2)" + ); + } } mod boundary { @@ -1329,12 +1396,13 @@ mod boundary { .close_frame_and_batch(&mut head, 100) .expect("close batch"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 1299, &[StoredSafeInput { sender: SENDER_A, - payload: make_stale_batch_payload(0, 100), + payload: landed, block_number: 1299, }], SENDER_A, @@ -1469,12 +1537,13 @@ mod boundary { storage .close_frame_and_batch(&mut head3, 2410) .expect("close gen3"); + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 2420, &[StoredSafeInput { sender: SENDER_A, - payload: make_stale_batch_payload(0, 2410), + payload: landed, block_number: 2420, }], SENDER_A, @@ -1581,6 +1650,8 @@ mod schema_invariants { #[test] fn schema_rejects_genesis_with_nonzero_nonce() { + // Default anchor is 0, so a parentless root at a non-zero nonce ABORTs + // (genesis must be 0) — the exact-match form of the contiguity rule. let db = temp_db("schema-genesis-nonzero"); let storage = Storage::open(db.path.as_str()).expect("open storage"); let err = storage.conn.execute( @@ -1589,8 +1660,119 @@ mod schema_invariants { [], ); assert!( - format!("{err:?}").contains("genesis batch must have nonce 0"), - "expected genesis-nonce trigger, got: {err:?}" + format!("{err:?}").contains("parentless root must carry the batch-tree anchor nonce"), + "expected anchor-nonce trigger, got: {err:?}" + ); + } + + #[test] + fn schema_allows_parentless_root_at_anchor_nonce() { + // With the anchor set to N' (a recovered deployment), a parentless root + // at N' is permitted — the recovery generalization of "genesis at 0". + // (Rows are inserted sealed so the single-Tip partial index, which only + // covers open rows, doesn't mask the contiguity trigger under test.) + let db = temp_db("schema-anchor-root"); + let storage = Storage::open(db.path.as_str()).expect("open storage"); + storage + .conn + .execute( + "UPDATE batch_tree_anchor SET nonce = 42 WHERE singleton_id = 0", + [], + ) + .expect("set anchor"); + // A root at the anchor nonce (42) is accepted... + storage + .conn + .execute( + "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ + VALUES (0, NULL, 42, 100, 100)", + [], + ) + .expect("parentless root at the anchor nonce is allowed"); + // ...but a root at 0 (the old hard-coded value) now ABORTs — the rule is + // an exact match against the anchor, tighter than the old "must be 0". + let err = storage.conn.execute( + "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ + VALUES (1, NULL, 0, 100, 100)", + [], + ); + assert!( + format!("{err:?}").contains("parentless root must carry the batch-tree anchor nonce"), + "a root at the wrong nonce must abort, got: {err:?}" + ); + } + + #[test] + fn schema_rejects_second_valid_parentless_root() { + // At most one *valid* parentless root: a second parentless insert (both + // valid, both at the anchor nonce 0) ABORTs on the single-root guard. + // (Sealed inserts isolate this from the single-Tip index; a fully-torn + // cascade re-roots only after invalidating the old root, covered below.) + let db = temp_db("schema-two-roots"); + let storage = Storage::open(db.path.as_str()).expect("open storage"); + storage + .conn + .execute( + "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ + VALUES (0, NULL, 0, 100, 100)", + [], + ) + .expect("first root ok"); + let err = storage.conn.execute( + "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ + VALUES (1, NULL, 0, 100, 100)", + [], + ); + assert!( + format!("{err:?}").contains("at most one valid parentless root"), + "expected single-root guard, got: {err:?}" + ); + // Invalidate the first root, then a second parentless root is allowed — + // exactly the fully-torn-cascade re-root shape. + storage + .conn + .execute( + "UPDATE batches SET invalidated_at_ms = 200 WHERE batch_index = 0", + [], + ) + .expect("invalidate first root"); + storage + .conn + .execute( + "INSERT INTO batches (batch_index, parent_batch_index, nonce, created_at_ms, sealed_at_ms) \ + VALUES (1, NULL, 0, 100, 100)", + [], + ) + .expect("re-root parentless after the old root is invalidated"); + } + + #[test] + fn schema_freezes_anchor_after_setup_complete() { + // The anchor is write-once: once `setup_complete` exists, UPDATEs abort. + let db = temp_db("schema-anchor-frozen"); + let storage = Storage::open(db.path.as_str()).expect("open storage"); + // Pre-marker: UPDATE is allowed (this is what recovery setup does). + storage + .conn + .execute( + "UPDATE batch_tree_anchor SET nonce = 7 WHERE singleton_id = 0", + [], + ) + .expect("anchor settable before setup_complete"); + storage + .conn + .execute( + "INSERT INTO setup_complete (singleton_id, completed_at_ms) VALUES (0, 1)", + [], + ) + .expect("mark setup complete"); + let err = storage.conn.execute( + "UPDATE batch_tree_anchor SET nonce = 8 WHERE singleton_id = 0", + [], + ); + assert!( + format!("{err:?}").contains("batch-tree anchor is frozen after setup completes"), + "expected anchor-freeze trigger, got: {err:?}" ); } @@ -1700,6 +1882,80 @@ mod schema_invariants { ); } + #[test] + fn schema_rejects_sequenced_l2_tx_into_non_tip() { + // The third sibling of the tip-only triggers (frames + user_ops already + // have negative tests; sequenced_l2_txs did not). The global replay order + // is the source for recovery re-drain and catch-up; a stale-WriteHead row + // into a sealed batch would corrupt it. + let db = temp_db("schema-sequenced-into-sealed"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 0) + .expect("close batch 0; it is now sealed (no longer the Tip)"); + // Target the sealed batch's existing frame (0, 0). The BEFORE-INSERT + // trigger fires before the safe_input_index FK is evaluated. + let err = storage.conn.execute( + "INSERT INTO sequenced_l2_txs \ + (offset, batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ + VALUES (999, 0, 0, NULL, 0)", + [], + ); + assert!( + format!("{err:?}").contains("sequenced_l2_txs can only target the current Tip"), + "expected tip-only-sequenced trigger, got: {err:?}" + ); + } + + #[test] + fn schema_rejects_nonce_mutation() { + // Nonce immutability underpins the frontier + contiguity invariants. + let db = temp_db("schema-nonce-immutable"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 0) + .expect("close batch 0"); + let err = storage.conn.execute( + "UPDATE batches SET nonce = nonce + 1 WHERE batch_index = 0", + [], + ); + assert!( + format!("{err:?}").contains("nonce is immutable"), + "expected nonce-immutable trigger, got: {err:?}" + ); + } + + #[test] + fn schema_rejects_payload_hash_rewrite() { + // payload_hash is the content-identity anchor for the R2 canonical- + // divergence check; a rewrite would let a foreign/zombie L1 landing + // false-match a local batch. Write-once once stamped at seal. + let db = temp_db("schema-payload-hash-write-once"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .expect("initialize"); + storage + .close_frame_and_batch(&mut head, 0) + .expect("close batch 0; payload_hash is stamped at seal"); + let err = storage.conn.execute( + "UPDATE batches SET payload_hash = \ + X'0000000000000000000000000000000000000000000000000000000000000000' \ + WHERE batch_index = 0", + [], + ); + assert!( + format!("{err:?}").contains("payload_hash is write-once"), + "expected payload-hash-write-once trigger, got: {err:?}" + ); + } + #[test] fn nonce_reuse_after_cascade_with_valid_ancestor() { // Beautiful part of parent-pointer + structural nonce: after a cascade @@ -1730,12 +1986,13 @@ mod schema_invariants { // Head is now batch 3 (nonce 3, first_frame_safe_block=100). // Batch 0 lands on L1 (accepted): safe_input at block 20 with nonce 0. + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -2037,12 +2294,13 @@ mod tree_invariants { // Tree: 0(Gold sentinel in concept)→1→2→3→4 (Tip) // Phase 2: cascade with a valid ancestor. Batch 0 is accepted first. + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -2101,12 +2359,13 @@ mod tree_invariants { .close_frame_and_batch(&mut head, 100) .expect("close"); } + let landed = local_batch_payload(&mut storage, 0); storage .append_safe_inputs( 20, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), + payload: landed, block_number: 20, }], SENDER_A, @@ -2337,4 +2596,55 @@ mod recovery_clears_pending_snapshots { "no-op recovery must preserve in-flight pending snapshots" ); } + + /// Review F9 regression: the cascade's pending clear is scoped to + /// `nonce >= pivot.nonce`. A gold-but-unpromoted pending (its batch + /// landed accepted while the process was down, the lane never + /// promoted it) sits *below* the pivot and must survive — deleting + /// it would arm a promote-wedge crash-loop when the lane later + /// observes the landing and promotion hits the deleted row. + #[test] + fn cascade_preserves_gold_unpromoted_pending_below_pivot() { + let db = temp_db("recovery-scoped-clear-preserves-gold"); + let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .expect("initialize"); + // Close two batches: nonce 0 (will land gold) and nonce 1 (doomed). + storage + .close_frame_and_batch(&mut head, 10) + .expect("close batch nonce 0"); + storage + .close_frame_and_batch(&mut head, 10) + .expect("close batch nonce 1"); + + register_finalized(&mut storage, "fin-scoped"); + register_pending(&mut storage, 0); // gold-but-unpromoted + register_pending(&mut storage, 1); // doomed (past the frontier) + + // Batch nonce 0 lands accepted: gold frontier advances to 1, so + // the post-flush cascade pivots at the nonce-1 batch. + let batch_submitter = Address::repeat_byte(0xAA); + seed_safe_inputs_with_batch_nonces(&mut storage, batch_submitter, 10, &[0]); + + let invalidated = storage + .recover_post_flush(1200) + .expect("post-flush recover"); + assert_eq!( + invalidated, + vec![1, 2], + "cascade covers the nonce-1 batch and the Tip" + ); + + let survivor = storage + .latest_pending_dump() + .unwrap() + .expect("gold-unpromoted pending must survive the scoped clear"); + assert_eq!(survivor.nonce, 0, "the survivor is the gold pending"); + assert!( + storage.finalized_dump().unwrap().is_some(), + "cascade must not touch finalized" + ); + } } diff --git a/sequencer/src/storage/safe_accepted_batches.rs b/sequencer/src/storage/safe_accepted_batches.rs index 3ec5f40..d62ef08 100644 --- a/sequencer/src/storage/safe_accepted_batches.rs +++ b/sequencer/src/storage/safe_accepted_batches.rs @@ -24,8 +24,9 @@ use alloy_primitives::Address; use rusqlite::{Connection, OptionalExtension, Result, params}; -use super::convert::{i64_to_u64, u64_to_i64}; -use sequencer_core::protocol::{ProtocolTiming, SafeInputView}; +use super::convert::{i64_to_u64, now_unix_ms, u64_to_i64}; +use super::mutations::batch_tree_anchor_in; +use sequencer_core::protocol::{AcceptedBatch, ProtocolTiming, SafeInputView}; /// One row of `safe_accepted_batches`, exposing just the columns the /// frontier-read code paths need. @@ -61,9 +62,21 @@ pub(super) fn query_latest_safe_accepted_batch( /// pivot, when one exists) will carry, by the contiguity invariant on the /// valid path (`trg_enforce_nonce_contiguity`). pub(super) fn frontier_nonce(conn: &Connection) -> Result { - Ok(query_latest_safe_accepted_batch(conn)? - .map(|row| i64_to_u64(row.nonce).saturating_add(1)) - .unwrap_or(0)) + match query_latest_safe_accepted_batch(conn)? { + Some(row) => Ok(i64_to_u64(row.nonce).saturating_add(1)), + // Empty accepted table ⇒ the frontier sits at the batch-tree anchor: 0 + // for a genesis deployment, N' for a cockroach-recovered one. Reading + // the anchor (not a hard-coded 0) keeps this in step with + // `populate_safe_accepted_batches`, which seeds `expected` from the same + // anchor. Without it, a freshly recovered deployment — whose frontier + // stays empty until `run`'s first sync lands an accepted row — would + // have its submitter fold from 0 and re-submit its first post-recovery + // batch (nonce N') every tick until then. The cascade reader + // (`first_non_gold_closed_batch`) is unaffected: no valid batch sits + // below the anchor (I16), so `nonce >= 0` and `nonce >= N'` select the + // same row. (See I16 / docs/recovery/cockroach.md.) + None => batch_tree_anchor_in(conn), + } } /// Simulate the scheduler's acceptance logic over new safe inputs and append @@ -89,6 +102,16 @@ pub(super) fn frontier_nonce(conn: &Connection) -> Result { /// batch-submitter inputs after the gold frontier can be rescanned on later /// safe-head syncs until a later batch is accepted and moves the accepted /// cursor forward. +/// +/// Scheduler-mirror co-location: the acceptance *predicate* lives at the +/// library seam ([`ProtocolTiming::scheduler_accepts`]), and the bare +/// nonce-advance fold in `protocol::advance_expected_batch_nonce`. +/// This loop deliberately keeps its own inline `expected` advance rather than +/// reusing that fold: the advance is interleaved with two storage-only side +/// effects that cannot move below the protocol layer — the R2 content-identity +/// check ([`content_identity_violation`]) and the `canonical_divergence` freeze. +/// Sharing a fold here would force a callback contract for those (a refactor, +/// not the no-behavior-change library move). pub(super) fn populate_safe_accepted_batches( conn: &Connection, batch_submitter: Address, @@ -103,13 +126,34 @@ pub(super) fn populate_safe_accepted_batches( (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ VALUES (?1, ?2, ?3, ?4)"; + // A persisted divergence marker freezes the acceptance frontier: the + // local batch tree is no longer a reliable mirror of canonical state, + // and advancing it (or promoting on it) would compound the divergence. + // `check_danger` reports `CanonicalDivergence` ahead of every other arm, + // so the detector exits / startup refuses; the remedy is cockroach + // recovery, never standard recovery (review R2). + if canonical_divergence_in(conn)?.is_some() { + return Ok(()); + } + + // The frontier begins at the batch-tree anchor: 0 for a genesis + // deployment (unchanged), or N' for a cockroach-recovered one. Below the + // anchor the local tree has no batches — that history is folded into the + // recovered checkpoint `S'`, not kept as tree batches — so those L1 + // landings are *trusted collapsed history*, not foreign. Seeding `expected` + // at the anchor makes the scan skip them by nonce-mismatch (they never + // reach the content-identity check), while landings at/above the anchor — + // the resumed instance's own batches — are accepted and checked normally. + // (See I16 / docs/recovery: the anchor is where the local tree's authority + // begins.) + let anchor = batch_tree_anchor_in(conn)?; let latest_accepted = query_latest_safe_accepted_batch(conn)?; let mut cursor = latest_accepted .map(|row| row.safe_input_index) .unwrap_or(-1); let mut expected = latest_accepted .map(|row| i64_to_u64(row.nonce).saturating_add(1)) - .unwrap_or(0); + .unwrap_or(anchor); loop { // Materialize one page before executing any INSERTs. rusqlite's row @@ -142,6 +186,27 @@ pub(super) fn populate_safe_accepted_batches( let Some(accepted) = timing.scheduler_accepts(batch_submitter, input, expected) else { continue; }; + + // Content-identity check (review R2), gated on full acceptance — + // exactly here, where the simulated scheduler accepted the + // landing. Rejected/stale/undecodable copies are scheduler + // no-ops; their content is irrelevant by construction. + if let Some(kind) = content_identity_violation(conn, &accepted, payload.as_slice())? { + record_canonical_divergence_in(conn, &accepted, kind)?; + tracing::error!( + nonce = accepted.nonce, + safe_input_index = accepted.safe_input_index, + kind = kind.as_str(), + "CANONICAL DIVERGENCE: accepted L1 landing does not match \ + our batch at this nonce; freezing the acceptance frontier \ + — remedy is cockroach recovery (wipe + rebuild from L1)" + ); + // Stop scanning; the marker (committed with this sync) + // freezes the frontier and routes every subsequent boot and + // detector tick to refusal. + return Ok(()); + } + insert_stmt.execute(params![ u64_to_i64(accepted.safe_input_index), u64_to_i64(accepted.nonce), @@ -158,3 +223,111 @@ pub(super) fn populate_safe_accepted_batches( Ok(()) } + +/// How an accepted landing failed the content-identity check. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DivergenceKind { + /// No valid closed local batch exists at the accepted nonce — we never + /// (validly) submitted one, so the landing is a zombie or foreign batch + /// from our key. + Foreign, + /// A valid closed local batch exists, but the landed wire bytes hash + /// differently from the bytes we sealed. + Mismatch, +} + +impl DivergenceKind { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Foreign => "foreign", + Self::Mismatch => "mismatch", + } + } +} + +/// R2 check proper: compare a fully-accepted landing against our valid +/// closed batch at the same nonce. `None` = identical (the normal case). +/// +/// Why content (not identity) suffices: accepted content-equal copies are +/// effect-equal — which physical tx landed carries no semantic weight. The +/// hash on our side was stamped at seal by the same encode path the +/// submitter broadcasts, so the comparison survives wire-format upgrades. +fn content_identity_violation( + conn: &Connection, + accepted: &AcceptedBatch, + landed_payload: &[u8], +) -> Result> { + let ours: Option>> = conn + .query_row( + "SELECT payload_hash FROM valid_closed_batches WHERE nonce = ?1", + params![u64_to_i64(accepted.nonce)], + |row| row.get(0), + ) + .optional()?; + match ours { + None => Ok(Some(DivergenceKind::Foreign)), + Some(None) => { + // Sealed batches always carry a hash (stamped in the seal + // UPDATE); a NULL here is an impossible state — fail loud. + // (Recovery sentinels carry no hash, but they sit *behind* the + // acceptance frontier by construction and are never compared.) + // Surface it as a structured error (rolls back the reader's sync + // transaction and travels its normal error path) rather than a + // panic that would crash the reader task mid-transaction. + Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CORRUPT), + Some(format!( + "valid closed batch at nonce {} has no payload_hash", + accepted.nonce + )), + )) + } + Some(Some(hash)) => { + let landed = alloy_primitives::keccak256(landed_payload); + if hash.as_slice() == landed.as_slice() { + Ok(None) + } else { + Ok(Some(DivergenceKind::Mismatch)) + } + } + } +} + +/// The persisted divergence marker, if any: `(nonce, safe_input_index)`. +pub(super) fn canonical_divergence_in(conn: &Connection) -> Result> { + conn.query_row( + "SELECT nonce, safe_input_index FROM canonical_divergence WHERE singleton_id = 0", + [], + |row| { + Ok(( + i64_to_u64(row.get::<_, i64>(0)?), + i64_to_u64(row.get::<_, i64>(1)?), + )) + }, + ) + .optional() +} + +/// Persist the poison marker, atomically with the sync that detected it +/// (same transaction). Keep-first by construction: the frontier freezes at +/// the first detection, so this can only ever run once per marker lifetime +/// (the guard at the top of `populate_safe_accepted_batches`); a second +/// INSERT would fail loud on the singleton PK. +fn record_canonical_divergence_in( + conn: &Connection, + accepted: &AcceptedBatch, + kind: DivergenceKind, +) -> Result<()> { + conn.execute( + "INSERT INTO canonical_divergence \ + (singleton_id, nonce, safe_input_index, kind, detected_at_ms) \ + VALUES (0, ?1, ?2, ?3, ?4)", + params![ + u64_to_i64(accepted.nonce), + u64_to_i64(accepted.safe_input_index), + kind.as_str(), + now_unix_ms(), + ], + )?; + Ok(()) +} diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index 4ebfaff..a56cd4e 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -309,11 +309,12 @@ impl Storage { self.read(list_dump_rows_in) } - /// Delete every row from `pending_snapshots`. Test-only convenience - /// wrapper: production danger-zone recovery composes - /// `clear_pending_dumps_in` into the same transaction as the cascade - /// invalidation (see `storage/recovery.rs`), so the pending rows for - /// cascade-doomed batches are cleared atomically with them. + /// Delete every row from `pending_snapshots`. Test-only convenience wrapper + /// for the *unscoped* clear: production danger-zone recovery instead composes + /// the pivot-scoped `clear_pending_dumps_from_nonce_in` (F9) into the same + /// transaction as the cascade invalidation (see `storage/recovery.rs`), so + /// only the cascade-doomed batches' pending rows are cleared, atomically with + /// them. #[cfg(test)] pub fn clear_pending_dumps(&mut self) -> Result { self.write(clear_pending_dumps_in) @@ -350,12 +351,11 @@ impl Storage { } /// Highest `offset` in the valid ordered L2-tx stream (the global - /// replay head), or 0 when empty. Test-only standalone read: the - /// production batch-close path reads the same value via the - /// `valid_ordered_l2_tx_head` free function *inside* its seal - /// transaction (see `close_frame_and_batch_with_pending_dump`), so - /// the recorded `l2_tx_index` is consistent with the seal. - #[cfg(test)] + /// replay head), or 0 when empty. The lane reads this before writing + /// a dump's `info.toml` at batch close; the seal transaction + /// re-asserts the same value (see + /// `close_frame_and_batch_with_pending_dump`), which is sound because + /// the lane is the single writer and nothing sequences in between. pub fn valid_ordered_l2_tx_head(&mut self) -> Result { self.read(|tx| super::queries::valid_ordered_l2_tx_head(tx)) } @@ -365,14 +365,7 @@ impl Storage { /// calls this immediately after `close_frame_and_batch` so the row /// should always be there. pub fn batch_nonce(&mut self, batch_index: u64) -> Result { - self.read(|tx| { - let nonce: i64 = tx.query_row( - "SELECT nonce FROM batches WHERE batch_index = ?1", - params![u64_to_i64(batch_index)], - |row| row.get(0), - )?; - Ok(i64_to_u64(nonce)) - }) + self.read(|tx| batch_nonce_in(tx, batch_index)) } /// Look up the nonce of a previously-accepted batch by its safe @@ -567,10 +560,39 @@ fn list_dump_rows_in(tx: &Transaction<'_>) -> Result> { /// the same transaction as `cascade_invalidate_from` — otherwise a /// crash between the cascade and the clear would leave stale pending /// snapshots pointing at states the canonical stream will never reach. +#[cfg(test)] pub(super) fn clear_pending_dumps_in(tx: &Transaction<'_>) -> Result { tx.execute("DELETE FROM pending_snapshots", []) } +/// Scoped pending-snapshot clear for the recovery cascade: delete only the +/// rows whose `nonce >= from_nonce` (the cascade pivot's nonce) — exactly +/// the cascaded batches' pendings. Lower-nonce rows are gold-but-unpromoted +/// pendings that must survive (review F9: deleting them arms a +/// promote-wedge crash-loop when their landing is later observed). Same +/// same-transaction composition rationale as [`clear_pending_dumps_in`]. +pub(super) fn clear_pending_dumps_from_nonce_in( + tx: &Transaction<'_>, + from_nonce: u64, +) -> Result { + tx.execute( + "DELETE FROM pending_snapshots WHERE nonce >= ?1", + params![u64_to_i64(from_nonce)], + ) +} + +/// Free-function form of [`Storage::batch_nonce`] for composing into a +/// larger transaction (the recovery cascade reads its pivot's nonce to +/// scope the pending clear). +pub(super) fn batch_nonce_in(conn: &rusqlite::Connection, batch_index: u64) -> Result { + let nonce: i64 = conn.query_row( + "SELECT nonce FROM batches WHERE batch_index = ?1", + params![u64_to_i64(batch_index)], + |row| row.get(0), + )?; + Ok(i64_to_u64(nonce)) +} + fn row_to_dump_row(row: &rusqlite::Row<'_>) -> Result { Ok(DumpRow { id: row.get(0)?, diff --git a/sequencer/src/storage/test_helpers.rs b/sequencer/src/storage/test_helpers.rs index 3ba67df..71393b9 100644 --- a/sequencer/src/storage/test_helpers.rs +++ b/sequencer/src/storage/test_helpers.rs @@ -42,10 +42,34 @@ pub(crate) fn temp_db(name: &str) -> TestDb { } } +/// Wire bytes of the local valid closed batch at `nonce` — the +/// production-faithful "our batch landed on L1" payload. Hash-matches the +/// seal-time stamp, so the content-identity check (review R2) accepts it. +/// Panics if no valid closed local batch carries `nonce` (test bug: an +/// accepted landing without a matching local batch is, by design, a +/// canonical divergence). +pub(crate) fn local_batch_payload(storage: &mut Storage, nonce: u64) -> Vec { + storage + .pending_batches(nonce) + .expect("load local closed batches") + .into_iter() + .find(|b| b.nonce == nonce) + .unwrap_or_else(|| panic!("no valid closed local batch at nonce {nonce}")) + .encoded +} + /// Insert safe inputs whose payloads are SSZ-encoded batches with the given /// nonces, all attributed to `sender`. `sender` doubles as the /// batch-submitter address passed to `append_safe_inputs`, so the populated /// `safe_accepted_batches` view matches this sender. +/// +/// Nonces with a valid closed local batch land with that batch's real wire +/// bytes (the landing will be *accepted* — the content-identity check +/// requires byte equality with the seal-time hash). Nonces without one get +/// a synthetic empty-batch payload — sound **only** for landings the +/// acceptance fold rejects (e.g. past a nonce gap); a synthetic landing +/// that would be accepted trips the divergence marker and freezes the +/// frontier. pub(crate) fn seed_safe_inputs_with_batch_nonces( storage: &mut Storage, sender: Address, @@ -54,13 +78,24 @@ pub(crate) fn seed_safe_inputs_with_batch_nonces( ) { let inputs: Vec = nonces .iter() - .map(|nonce| StoredSafeInput { - sender, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: *nonce, - frames: Vec::new(), - }), - block_number: safe_block, + .map(|nonce| { + let payload = storage + .pending_batches(*nonce) + .expect("load local closed batches") + .into_iter() + .find(|b| b.nonce == *nonce) + .map(|b| b.encoded) + .unwrap_or_else(|| { + ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { + nonce: *nonce, + frames: Vec::new(), + }) + }); + StoredSafeInput { + sender, + payload, + block_number: safe_block, + } }) .collect(); storage @@ -93,7 +128,7 @@ pub(crate) fn all_ordered_l2_txs(storage: &mut Storage) -> Vec { .ordered_l2_txs_page_from(0, 1_000_000) .expect("load all ordered l2 txs") .into_iter() - .map(|(_offset, tx)| tx) + .map(|(_offset, tx, _frame_safe_block)| tx) .collect() } diff --git a/sequencer/tests/batch_submitter_integration.rs b/sequencer/tests/batch_submitter_integration.rs index 930a713..c2906b6 100644 --- a/sequencer/tests/batch_submitter_integration.rs +++ b/sequencer/tests/batch_submitter_integration.rs @@ -10,6 +10,7 @@ use std::time::Duration; use async_trait::async_trait; use sequencer::l1::submitter::{BatchPoster, BatchPosterError, TxHash}; use sequencer::l1::submitter::{BatchSubmitter, BatchSubmitterConfig}; +use sequencer::l1::watermark::WalletNonceWatermarkSink; use sequencer::runtime::shutdown::ShutdownSignal; use sequencer::storage::{SafeInputRange, Storage}; use sequencer_core::batch::Batch; @@ -60,6 +61,7 @@ impl BatchPoster for TestMock { async fn submit_batches( &self, payloads: Vec>, + _watermark: &dyn WalletNonceWatermarkSink, ) -> Result, BatchPosterError> { // Transient-failure hook: consume one of the configured failures // before anything else, so the tick outcome maps to `Transient` and diff --git a/sequencer/tests/chain_id_validation.rs b/sequencer/tests/chain_id_validation.rs index 1ce77aa..7dc60dc 100644 --- a/sequencer/tests/chain_id_validation.rs +++ b/sequencer/tests/chain_id_validation.rs @@ -1,35 +1,39 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! H7 regression: chain-id/deployment mismatch is caught early in bootstrap. +//! `run`-side bootstrap guards after the setup/run split. //! -//! The H7 hardening moved the live RPC chain-id check before deployment -//! identity writes and replaced `assert_eq!` with typed bootstrap errors. This -//! file locks two code paths where the check matters: +//! `run` no longer takes `--chain-id` / `--app-address` — it reads the pinned +//! deployment identity from the DB that `setup` created. These tests lock the +//! refusal paths that protect that handoff, without needing the on-chain +//! contracts that `setup`'s L1 discovery requires (those live in the full +//! rollups-e2e harness): //! -//! - Identity path: L1 is unreachable but a deployment identity exists with -//! a different chain_id. Check fires before `InputReader::from_parts`. -//! - Positive control: with a matched chain_id, `ChainIdMismatch` does NOT -//! fire, so the check doesn't misfire on the happy path. -//! -//! The RPC path (L1 reachable, chain_id from `eth_chainId` mismatches) is -//! NOT covered here because `InputReader::new` needs a real InputBox contract -//! deployed at `config.app_address` before the chain-id check fires. That -//! setup only exists in the full rollups-e2e harness (after `just setup`) — -//! see `chain_id_mismatch_via_live_rpc_refuses_boot_test` there. +//! - **No setup** → `BootstrapError::SetupNotComplete` (terminal; operator +//! must run `setup`). Fires before any L1 contact. +//! - **Wrong signing key** → `IdentityError::Mismatch { batch_submitter_address }` +//! (the key's address must match the pinned submitter). Fires before L1. +//! - **Wrong-chain RPC** (review F6) → `ChainIdMismatch`: a reachable RPC +//! whose `eth_chainId` differs from the pinned chain id is refused. Needs +//! a live RPC, so it uses Anvil. +//! - **Matching-chain RPC** positive control: a matching chain must NOT +//! produce `ChainIdMismatch`. use std::time::Duration; use alloy_primitives::{Address, address}; -use app_core::application::{WalletApp, WalletConfig}; use clap::Parser; -use sequencer::RunConfig; use sequencer::runtime::{BootstrapError, IdentityError, RunError}; +use sequencer::storage::{DeploymentIdentity, Storage}; +use sequencer::{Cli, Command}; use tempfile::TempDir; -// Anvil's default devnet private key #0. +// Anvil's default devnet private key #0 and its address. const ANVIL_KEY: &str = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; -const TEST_APP_ADDR: &str = "0x1111111111111111111111111111111111111111"; +const ANVIL_ADDR: Address = address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); +const OTHER_KEY: &str = "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const TEST_APP_ADDR: Address = address!("0x1111111111111111111111111111111111111111"); +const TEST_INPUT_BOX: Address = address!("0x2222222222222222222222222222222222222222"); /// Verify that `anvil` is available. Panics with a clear message if not found. fn require_anvil() { @@ -44,117 +48,165 @@ fn require_anvil() { ); } -fn build_config( - data_dir: &str, - eth_rpc_url: &str, - chain_id: u64, -) -> Result { - RunConfig::try_parse_from([ +/// Parse a `run` config through the harness CLI and extract it. +fn run_config(data_dir: &str, eth_rpc_url: &str, key: &str) -> sequencer::RunConfig { + let cli = Cli::try_parse_from([ "sequencer", + "run", "--http-addr", "127.0.0.1:0", "--data-dir", data_dir, "--eth-rpc-url", eth_rpc_url, - "--chain-id", - &chain_id.to_string(), - "--app-address", - TEST_APP_ADDR, "--batch-submitter-private-key", - ANVIL_KEY, + key, ]) + .expect("parse run config"); + match cli.command { + Command::Run(c) => *c, + other => panic!("expected run subcommand, got {other:?}"), + } } -fn build_app() -> WalletApp { - WalletApp::new(WalletConfig::default()) +/// Seed a pinned deployment identity (chain id `chain_id`, submitter +/// `submitter`) and mark setup complete — the minimal DB state `run` expects +/// from a completed `setup`, without `setup`'s L1 discovery. +fn seed_setup_complete(db_path: &str, chain_id: u64, submitter: Address) { + let mut storage = Storage::open(db_path).expect("open db for seed"); + storage + .load_or_insert_deployment_identity(DeploymentIdentity { + chain_id, + app_address: TEST_APP_ADDR, + input_box_address: TEST_INPUT_BOX, + input_box_genesis_block: 100, + batch_submitter_address: submitter, + }) + .expect("seed deployment identity"); + storage.mark_setup_complete().expect("mark setup complete"); } -// ── Deployment-identity fallback path ─────────────────────────────── - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn chain_id_mismatch_from_deployment_identity_returns_typed_error() { - // Scenario: L1 is unreachable, but a deployment identity exists from a - // previous successful run. The stored chain_id does NOT match the current - // config. The fallback arm must return a typed identity mismatch. - +async fn run_refuses_when_setup_incomplete() { + // Fresh DB, no marker: `run` must refuse before touching L1. let dir = TempDir::new().expect("tempdir"); let data_dir = dir.path().to_str().unwrap(); + let config = run_config(data_dir, "http://127.0.0.1:1", ANVIL_KEY); - // Pre-populate deployment identity with chain_id=31337. - let db_path = format!("{data_dir}/sequencer.db"); - { - let mut storage = sequencer::storage::Storage::open(&db_path).expect("open db for seed"); - storage - .load_or_insert_deployment_identity(sequencer::storage::DeploymentIdentity { - chain_id: 31_337, - app_address: address!("0x1111111111111111111111111111111111111111"), - input_box_address: Address::from_slice(&[0x22; 20]), - input_box_genesis_block: 100, - batch_submitter_address: address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"), - }) - .expect("seed deployment identity"); - } + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() must return quickly without a setup-complete marker"); - // Point the sequencer at an unreachable RPC (port 1, reliably refused) and - // a MISMATCHED chain_id=1. L1 is unreachable → identity-fallback path runs - // → stored chain_id (31337) mismatches config (1). - let config = build_config(data_dir, "http://127.0.0.1:1", 1).expect("parse config"); + assert!( + matches!( + result, + Err(RunError::Bootstrap(BootstrapError::SetupNotComplete)) + ), + "expected SetupNotComplete, got: {result:?}" + ); +} - let result = tokio::time::timeout(Duration::from_secs(30), sequencer::run(build_app(), config)) - .await - .expect("run() must return quickly on mismatch"); +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn run_refuses_on_submitter_key_mismatch() { + // Setup pinned ANVIL_ADDR as the submitter; running with OTHER_KEY + // (a different address) must fail loud before any L1 contact. + let dir = TempDir::new().expect("tempdir"); + let data_dir = dir.path().to_str().unwrap(); + let db_path = format!("{data_dir}/sequencer.db"); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + + let config = run_config(data_dir, "http://127.0.0.1:1", OTHER_KEY); + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() must return quickly on key/identity mismatch"); match result { Err(RunError::Bootstrap(BootstrapError::Identity(IdentityError::Mismatch { fields, - stored, - expected, - }))) => { - assert_eq!(fields, "chain_id"); - assert_eq!(stored.chain_id, 31_337); - assert_eq!(expected.chain_id, 1); - } - other => panic!("expected IdentityError::Mismatch, got: {other:?}"), + .. + }))) => assert_eq!(fields, "batch_submitter_address"), + other => panic!("expected batch_submitter_address mismatch, got: {other:?}"), } } -// ── Positive: matched chain_id does NOT trigger ChainIdMismatch ────────────── - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn chain_id_match_does_not_produce_mismatch_error() { - // Positive control: when chain_id matches, we should NOT get ChainIdMismatch. - // (The sequencer then tries to start the full stack. We don't care about - // that — a timeout counts as "didn't return ChainIdMismatch early", which - // is what we want to verify.) +async fn run_refuses_on_wrong_chain_rpc() { + // Pinned chain id 31337, but the (reachable) RPC reports a different + // chain id — review F6: a wrong-chain RPC after setup must be refused. require_anvil(); - - let anvil = alloy::node_bindings::Anvil::default().spawn(); - let rpc_url = anvil.endpoint(); + let anvil = alloy::node_bindings::Anvil::default().chain_id(99).spawn(); let dir = TempDir::new().expect("tempdir"); - let config = build_config(dir.path().to_str().unwrap(), &rpc_url, 31_337) - .expect("parse config with matching chain_id"); - - // Short timeout: if ChainIdMismatch is going to fire, it fires fast. - // A timeout means the check passed and the sequencer is running normally. - let result = - tokio::time::timeout(Duration::from_secs(3), sequencer::run(build_app(), config)).await; + let data_dir = dir.path().to_str().unwrap(); + let db_path = format!("{data_dir}/sequencer.db"); + // Submitter = anvil key address so the key check passes and we reach the + // chain-id check. + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + + let config = run_config(data_dir, &anvil.endpoint(), ANVIL_KEY); + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() must return quickly on chain-id mismatch"); match result { - Err(_timeout) => {} // expected — sequencer is running - Ok(Err(RunError::Bootstrap(BootstrapError::ChainIdMismatch { rpc, config }))) => { - panic!( - "matched chain_id must not produce ChainIdMismatch, got rpc={rpc} config={config}" - ); - } - Ok(Err(other)) => { - // Some other error is fine — we only care that it's not ChainIdMismatch. - eprintln!( - "sequencer returned non-mismatch error (expected under test conditions): {other:?}" - ); - } - Ok(Ok(())) => { - panic!("sequencer should not complete run() in a short test window"); + Err(RunError::Bootstrap(BootstrapError::ChainIdMismatch { rpc, config })) => { + assert_eq!(rpc, 99); + assert_eq!(config, 31_337); } + other => panic!("expected ChainIdMismatch, got: {other:?}"), } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn run_accepts_matching_chain_rpc() { + // Positive control: a matching chain id must NOT produce ChainIdMismatch. + // The DB has identity + marker but no genesis snapshot, so `run` passes + // the chain-id check and then refuses at the always-load gate with + // SetupNotComplete — a deterministic proof the chain-id guard let it + // through (the gate sits immediately after the chain-id check, before any + // recovery write). + require_anvil(); + let anvil = alloy::node_bindings::Anvil::default().spawn(); // chain id 31337 + let dir = TempDir::new().expect("tempdir"); + let data_dir = dir.path().to_str().unwrap(); + let db_path = format!("{data_dir}/sequencer.db"); + seed_setup_complete(&db_path, 31_337, ANVIL_ADDR); + + let config = run_config(data_dir, &anvil.endpoint(), ANVIL_KEY); + let result = tokio::time::timeout( + Duration::from_secs(10), + sequencer::run::(config), + ) + .await + .expect("run() returns promptly: chain-id passes, then the no-snapshot gate fires"); + + // Assert the chain-id check positively did NOT fail, independent of which + // gate fires next: if `seed_setup_complete` ever starts registering a + // genesis snapshot, the `SetupNotComplete` arm below would stop firing and + // silently stop witnessing "chain id passed" — this negative pins it. + assert!( + !matches!( + result, + Err(RunError::Bootstrap( + BootstrapError::ChainIdMismatch { .. } | BootstrapError::ChainIdRpc { .. } + )) + ), + "matching chain id must not produce a chain-id error, got: {result:?}" + ); + assert!( + matches!( + result, + Err(RunError::Bootstrap(BootstrapError::SetupNotComplete)) + ), + "matching chain id must pass the chain-id check and reach the \ + always-load gate (SetupNotComplete), got: {result:?}" + ); +} diff --git a/sequencer/tests/e2e_sequencer.rs b/sequencer/tests/e2e_sequencer.rs index b61795c..eaaa5e5 100644 --- a/sequencer/tests/e2e_sequencer.rs +++ b/sequencer/tests/e2e_sequencer.rs @@ -15,7 +15,7 @@ use k256::ecdsa::signature::hazmat::PrehashSigner; use sequencer::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; use sequencer::http::{self, ApiConfig}; use sequencer::ingress::inclusion_lane::{ - InclusionLane, InclusionLaneConfig, InclusionLaneError, PendingUserOp, + InclusionLane, InclusionLaneConfig, InclusionLaneError, PendingUserOp, dump_info, }; use sequencer::runtime::shutdown::ShutdownSignal; use sequencer::storage::{SafeInputRange, Storage, StoredSafeInput}; @@ -846,14 +846,14 @@ async fn api_accepts_user_op_with_max_fee_equal_to_current_frame_fee() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn api_rejects_user_op_when_balance_below_gas_cost() { +async fn api_rejects_user_op_when_balance_below_fee_cost() { // if sender's balance < `fee_to_linear(current_frame_fee)` the - // user op must be rejected with 422 `InsufficientGasBalance` and leave + // user op must be rejected with 422 `InsufficientFeeBalance` and leave // state unchanged. Exercises the balance check in // `WalletApp::validate_user_op` (app-core). A fresh sender with no // deposits has balance 0, well below `fee_to_linear(1060)` (the // bootstrapped frame fee). - let db = temp_db("insufficient-gas-balance"); + let db = temp_db("insufficient-fee-balance"); let domain = test_domain(); let signing_key = SigningKey::from_bytes((&[11_u8; 32]).into()).expect("create signing key"); let sender = address_from_signing_key(&signing_key); @@ -891,8 +891,8 @@ async fn api_rejects_user_op_when_balance_below_gas_cost() { "insufficient-balance must produce 422, got {status}: {body}", ); assert!( - body.contains("insufficient balance for gas"), - "expected InsufficientGasBalance message, got: {body}", + body.contains("insufficient balance for fee"), + "expected InsufficientFeeBalance message, got: {body}", ); shutdown_runtime(runtime).await; @@ -1094,10 +1094,20 @@ async fn start_full_server_with_max_body( // The lane reloads via `from_dump` either way. if storage.finalized_dump().expect("read finalized").is_none() { let app = WalletApp::new(WalletConfig::default()); - let genesis_prefix = dumps_dir.join("genesis"); - app.create_dump(&genesis_prefix).expect("genesis dump"); + let genesis_dir = dumps_dir.join("genesis"); + dump_info::create_dump_dir_with_info( + &app, + &genesis_dir, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + l2_tx_index: 0, + promoted_inclusion_block: Some(0), + }, + ) + .expect("genesis dump"); storage - .insert_finalized_dump(&genesis_prefix, 0, 0) + .insert_finalized_dump(&genesis_dir, 0, 0) .expect("register genesis"); } @@ -1144,7 +1154,9 @@ async fn start_full_server_with_max_body( }, http::SnapshotState { db_path: db_path.to_string(), - state_file_in_dump: WalletApp::state_file_in_dump, + state_file_in_dump: |dump_dir| { + WalletApp::state_file_in_dump(&dump_info::app_prefix(dump_dir)) + }, }, ); @@ -1198,7 +1210,9 @@ async fn start_api_only_server( }, http::SnapshotState { db_path: db_path.to_string(), - state_file_in_dump: WalletApp::state_file_in_dump, + state_file_in_dump: |dump_dir| { + WalletApp::state_file_in_dump(&dump_info::app_prefix(dump_dir)) + }, }, ); @@ -1337,7 +1351,7 @@ fn all_ordered_l2_txs(db_path: &str) -> Vec { .ordered_l2_txs_page_from(0, 1_000_000) .expect("load ordered l2 txs") .into_iter() - .map(|(_offset, tx)| tx) + .map(|(_offset, tx, _frame_safe_block)| tx) .collect() } diff --git a/sequencer/tests/snapshot_endpoints.rs b/sequencer/tests/snapshot_endpoints.rs index c0c3c0c..840541d 100644 --- a/sequencer/tests/snapshot_endpoints.rs +++ b/sequencer/tests/snapshot_endpoints.rs @@ -19,6 +19,7 @@ use futures_util::StreamExt; use sequencer::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; use sequencer::http::{self, ApiConfig}; use sequencer::ingress::inclusion_lane::PendingUserOp; +use sequencer::ingress::inclusion_lane::dump_info; use sequencer::runtime::shutdown::ShutdownSignal; use sequencer::storage::Storage; use sequencer_core::application::Application; @@ -78,7 +79,9 @@ async fn start_server(db_path: &str) -> Option { ApiConfig::default(), http::SnapshotState { db_path: db_path.to_string(), - state_file_in_dump: WalletApp::state_file_in_dump, + state_file_in_dump: |dump_dir| { + WalletApp::state_file_in_dump(&dump_info::app_prefix(dump_dir)) + }, }, ); Some(TestServer { @@ -89,11 +92,24 @@ async fn start_server(db_path: &str) -> Option { }) } +/// Build a structured dump dir mirroring production layout: the app's +/// state under `state`, plus a minimal `info.toml`. fn write_state(dir: &Path, name: &str, bytes: &[u8]) -> std::path::PathBuf { - let prefix = dir.join(name); - std::fs::create_dir_all(&prefix).expect("mkdir dump"); - std::fs::write(WalletApp::state_file_in_dump(&prefix), bytes).expect("write state file"); - prefix + let dump_dir = dir.join(name); + let app_prefix = dump_info::app_prefix(&dump_dir); + std::fs::create_dir_all(&app_prefix).expect("mkdir dump"); + std::fs::write(WalletApp::state_file_in_dump(&app_prefix), bytes).expect("write state file"); + dump_info::write_info( + &dump_dir, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + l2_tx_index: 0, + promoted_inclusion_block: None, + }, + ) + .expect("write info.toml"); + dump_dir } fn register_finalized( @@ -460,8 +476,10 @@ async fn finalized_state_file_open_failure_releases_lease() { let dump_id = register_finalized(db.path.as_str(), dir.path(), "fin", b"bytes", 1, 1); // Delete the state file out from under the registered row to force the // post-acquire `File::open` error path. - std::fs::remove_file(WalletApp::state_file_in_dump(&dir.path().join("fin"))) - .expect("remove state file"); + std::fs::remove_file(WalletApp::state_file_in_dump(&dump_info::app_prefix( + &dir.path().join("fin"), + ))) + .expect("remove state file"); let Some(server) = start_server(db.path.as_str()).await else { return; }; diff --git a/sequencer/tests/ws_broadcaster.rs b/sequencer/tests/ws_broadcaster.rs index c6a6c6d..2c7a0d7 100644 --- a/sequencer/tests/ws_broadcaster.rs +++ b/sequencer/tests/ws_broadcaster.rs @@ -546,7 +546,7 @@ fn load_ordered_l2_txs_page(db_path: &str, from_offset: u64, limit: usize) -> Ve .ordered_l2_txs_page_from(from_offset, limit) .expect("load ordered l2 tx page") .into_iter() - .map(|(_offset, tx)| tx) + .map(|(_offset, tx, _frame_safe_block)| tx) .collect() } diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index c85afef..8e68c62 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -23,54 +23,62 @@ From repository root: ```bash just setup -just --justfile tests/benchmarks/justfile bench-unit just --justfile tests/benchmarks/justfile bench-ack-self just --justfile tests/benchmarks/justfile bench-round-trip-self -just --justfile tests/benchmarks/justfile bench-hammer-self just --justfile tests/benchmarks/justfile bench-sweep-self +just --justfile tests/benchmarks/justfile bench-rt-sweep-self +just --justfile tests/benchmarks/justfile bench-capacity-sweep-self +just --justfile tests/benchmarks/justfile bench-report just --justfile tests/benchmarks/justfile bench-compare-latest just --justfile tests/benchmarks/justfile all just --justfile tests/benchmarks/justfile all-and-compare ``` +The `just` recipes default `max_fee=1200`, which is above the placeholder app's +base fee. A run whose `--max-fee` is below the base fee has **every** tx +rejected (`422 EXECUTION_REJECTED: "max fee N below base fee ..."`) and reports +no accepted txs — set a fee at or above the base fee. + Direct `cargo` examples: ```bash -cargo run -p benchmarks --bin unit_hot_path -- --count 10000 --max-fee 0 -cargo run -p benchmarks --bin ack_latency -- --self-contained --count 200 --max-fee 0 --concurrency 1 -cargo run -p benchmarks --bin round_trip_latency -- --self-contained --count 100 --max-fee 0 --from-offset 0 --concurrency 1 -cargo run -p benchmarks --bin ack_latency -- --self-contained --count 5000 --max-fee 0 --concurrency 32 --evaluate -cargo run -p benchmarks --bin round_trip_latency -- --self-contained --count 5000 --max-fee 0 --from-offset 0 --concurrency 16 --evaluate -cargo run -p benchmarks --bin ack_latency -- --endpoint http://127.0.0.1:3000 --domain-chain-id 31337 --domain-verifying-contract 0x1111111111111111111111111111111111111111 --count 200 --max-fee 0 --concurrency 1 -cargo run -p benchmarks --bin round_trip_latency -- --endpoint http://127.0.0.1:3000 --domain-chain-id 31337 --domain-verifying-contract 0x1111111111111111111111111111111111111111 --count 100 --max-fee 0 --from-offset 0 --concurrency 1 -cargo run -p benchmarks --bin sweep -- --self-contained --mode round-trip --count 1000 --max-fee 0 --from-offset 0 --concurrency-list "1 2 4 8 16 32 64 96 128" -cargo run -p benchmarks --bin compare_latest --release -- --results-dir tests/benchmarks/results --kind all --sweep-mode round-trip +# Round-trip latency, self-contained (spawns anvil + sequencer). Runs are +# time-bounded (`--duration-secs`), not count-bounded. +cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 30 --concurrency 4 --max-fee 1200 +cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 60 --concurrency 16 --max-fee 1200 --evaluate + +# Against an external sequencer (pass that deployment's EIP-712 domain): +cargo run -p benchmarks --bin round_trip_latency --release -- --endpoint http://127.0.0.1:3000 --domain-chain-id 31337 --domain-verifying-contract 0x1111111111111111111111111111111111111111 --duration-secs 30 --concurrency 4 --max-fee 1200 + +# Concurrency sweep — ack latency by default, round-trip with `--round-trip`: +cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs 30 --max-fee 1200 --concurrency-list "1 2 4 8 16 32 64 128" +cargo run -p benchmarks --bin sweep --release -- --round-trip --self-contained --duration-secs 30 --max-fee 1200 --concurrency-list "1 2 4 8" + +# Aggregate the JSON artifacts / compare the two latest of a kind: +cargo run -p benchmarks --bin report --release -- --results-dir tests/benchmarks/results +cargo run -p benchmarks --bin compare_latest --release -- --results-dir tests/benchmarks/results --kind round-trip ``` ## Benchmarks -- `unit_hot_path`: measures local signing plus request JSON encoding. -- `ack_latency`: measures `POST /tx` acknowledgement latency for accepted txs. -- `round_trip_latency`: measures submit-to-broadcast latency (`POST /tx` to matching `GET /ws/subscribe` event) for accepted txs. -- `--evaluate` on `ack_latency` and `round_trip_latency`: prints a first-class target verdict block and stores it in JSON output. Today the verdict is expected to be `NOT_EVALUATED` because the harness only supports the same-host baseline, not the canonical network-aware profile from the spec. -- `bench-hammer`: high-concurrency round-trip run that verifies each accepted tx is observed on WS. -- `bench-sweep`: runs a concurrency sweep and emits a CSV plus capacity summary. Sweep reports separate: +- `round_trip_latency`: measures submit-to-broadcast latency (`POST /tx` to the matching `GET /ws/subscribe` event) for accepted txs. Drains existing WS backlog before timing so stale history does not pollute the window. +- `sweep`: runs a concurrency sweep — ack latency (`POST /tx` acknowledgement) by default, or round-trip with `--round-trip` — and emits a CSV plus capacity summary. Reports separate first-of-each markers: - first rejection of any kind - first HTTP non-`200` - first `429` - first client-side failure (`io_*`, timeouts, connection failures) -- `bench-compare-latest`: compares the latest two benchmark artifacts and prints deltas. Use `--sweep-mode ack|round-trip` to choose which sweep family to compare. -- `bench-soak-low-lat-self` and `bench-soak-high-throughput-self`: write timestamped JSON outputs by default so repeated runs do not overwrite previous soak artifacts. Pass `out=...` to force a specific path. +- `report`: aggregates the JSON artifacts under `--results-dir` into a summary. +- `compare_latest`: compares the two latest artifacts of a `--kind` (`ack`, `round-trip`, `rt-sweep`, `sweep`, `all`) and prints deltas. +- `--evaluate` on `round_trip_latency` / `sweep`: prints a first-class target verdict block and stores it in JSON output. Today the verdict is expected to be `NOT_EVALUATED` because the harness only supports the same-host baseline, not the canonical network-aware profile from the spec. ## Notes - Self-contained variants launch `anvil --load-state` from the preloaded rollups dump under `tests/benchmarks/.deps/`; run `just setup` first. - Self-contained variants also deploy a local `Application` through `ApplicationFactory`, so they require a canonical machine image at `examples/canonical-app/out/canonical-machine-image`; run `just canonical-build-machine-image` first. - Self-contained variants therefore require Foundry's `anvil` binary to be installed locally. -- Networked benches fail by default if any tx is rejected. Pass `--allow-rejections` to inspect mixed traffic. +- `--max-fee` must be at or above the placeholder app's base fee, or every tx is rejected (`422 EXECUTION_REJECTED`) and the run reports no accepted txs. The error message includes the rejection breakdown and the first rejection body, which names the base fee. - `round_trip_latency` drains existing WS backlog before timing so stale history does not pollute the measurement window. -- `bench-sweep mode=round-trip` carries `from_offset` forward across rounds to avoid re-reading old WS history. -- `--stop-on-first-non-200` now does exactly what it says: it stops on the first HTTP non-`200`, not on client-side transport failures. +- `sweep --round-trip` carries `from_offset` forward across rounds to avoid re-reading old WS history. - If sweep hits `Too many open files`, increase the shell limit (`ulimit -n 4096`) or use a smaller concurrency list. - Self-contained variants automatically build a temp DB, spawn `anvil`, start the sequencer, and persist logs/results under `tests/benchmarks/results`. - For non-self-contained runs, start a sequencer instance first and make sure the benchmark domain matches the sequencer domain. diff --git a/tests/benchmarks/justfile b/tests/benchmarks/justfile index ffb2ca0..3ac9079 100644 --- a/tests/benchmarks/justfile +++ b/tests/benchmarks/justfile @@ -27,26 +27,26 @@ ensure-machine-image: test -d {{template_machine_image}} || { echo "missing {{template_machine_image}}; run 'just canonical-build-machine-image' first"; exit 1; } bench-ack-self duration="45" max_fee="1200" extra="": ensure-machine-image - cargo build -p sequencer --release + cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list 1 --warmup-secs 5 --accounts-file {{accounts_file}} {{extra}} bench-round-trip-self duration="45" max_fee="1200" concurrency="1" extra="": ensure-machine-image - cargo build -p sequencer --release + cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency {{concurrency}} {{extra}} bench-sweep domain_chain_id verifying_contract duration="45" url="http://127.0.0.1:3000" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256" extra="": cargo run -p benchmarks --bin sweep --release -- --endpoint {{url}} --domain-chain-id {{domain_chain_id}} --domain-verifying-contract {{verifying_contract}} --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" {{extra}} bench-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256" extra="": ensure-machine-image - cargo build -p sequencer --release + cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" --accounts-file {{accounts_file}} {{extra}} bench-rt-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8" extra="": ensure-machine-image - cargo build -p sequencer --release + cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --round-trip --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" --accounts-file {{accounts_file}} {{extra}} bench-capacity-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256 512" out="tests/benchmarks/results/capacity-sweep-self.json" extra="": ensure-machine-image - cargo build -p sequencer --release + cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list {{conc_list}} --json-out {{out}} --accounts-file {{accounts_file}} {{extra}} bench-report results_dir="tests/benchmarks/results": diff --git a/tests/benchmarks/src/round_trip.rs b/tests/benchmarks/src/round_trip.rs index 3939d40..bada7c2 100644 --- a/tests/benchmarks/src/round_trip.rs +++ b/tests/benchmarks/src/round_trip.rs @@ -265,7 +265,16 @@ pub async fn run_round_trip_benchmark( let total_wall = started.elapsed(); if accepted_ack_samples.is_empty() { - return Err(std::io::Error::other("round-trip benchmark had no accepted txs").into()); + // Surface why every op was rejected — otherwise a misconfigured run + // (e.g. `--max-fee` below the app's base fee) reports a bare "no + // accepted txs" and looks like a sequencer fault. The breakdown + + // first rejection body point straight at the cause. + return Err(std::io::Error::other(format!( + "round-trip benchmark had no accepted txs \ + (rejected={rejected}, breakdown={rejection_breakdown:?}, \ + first_rejection={first_rejection:?})" + )) + .into()); } // Join submit starts with WS arrival timestamps to compute round-trip latencies. diff --git a/tests/benchmarks/src/runtime.rs b/tests/benchmarks/src/runtime.rs index 661c1d2..82f5e00 100644 --- a/tests/benchmarks/src/runtime.rs +++ b/tests/benchmarks/src/runtime.rs @@ -15,7 +15,7 @@ use crate::{BenchResult, BenchmarkDomain, WorkloadConfig}; pub use rollups_harness::ManagedSequencer; use rollups_harness::TestSigner; -pub const DEFAULT_SEQUENCER_BIN: &str = "target/release/sequencer-devnet"; +pub const DEFAULT_SEQUENCER_BIN: &str = "target/release/wallet-sequencer-devnet"; pub const DEFAULT_MEMORY_SAMPLE_INTERVAL_MS: u64 = 500; pub const DEFAULT_RESULTS_DIR: &str = "tests/benchmarks/results"; const BENCHMARK_FUNDING_SIGNER_PRIVATE_KEY: &str = diff --git a/tests/e2e/src/bin/devnet_stack.rs b/tests/e2e/src/bin/devnet_stack.rs index 65bce31..7894de5 100644 --- a/tests/e2e/src/bin/devnet_stack.rs +++ b/tests/e2e/src/bin/devnet_stack.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Local Anvil + rollups devnet + `sequencer-devnet` for manual watchdog runs. +//! Local Anvil + rollups devnet + `wallet-sequencer-devnet` for manual watchdog runs. //! //! Prints `CARTESI_WATCHDOG_*` exports, then blocks until Ctrl+C. diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index fa55467..b4b79fe 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -6,8 +6,9 @@ use std::time::Duration; use crate::{ScenarioFn, ScenarioResult}; use alloy_primitives::{Address, U256}; use rollups_harness::{ - ManagedSequencer, ReplayWalletApp, RespawnAttemptOutcome, RespawnPolicy, TcpProxy, TestSigner, - WalletL1Client, WalletL2Client, WsClient, sign_user_op_hex, + ManagedSequencer, RecoveryCheckpoint, RecoverySetupParams, ReplayWalletApp, + RespawnAttemptOutcome, RespawnPolicy, TcpProxy, TestSigner, WalletL1Client, WalletL2Client, + WsClient, sign_user_op_hex, }; use sequencer_core::api::{TxRequest, WsTxMessage}; use sequencer_core::fee::fee_to_linear; @@ -22,6 +23,21 @@ const DEFAULT_FRAME_FEE: u16 = 1060; /// Max fee used for raw TxRequest construction. Must be >= DEFAULT_FRAME_FEE. const DEFAULT_MAX_FEE: u16 = 1200; +/// Self-transfers that force the inclusion lane to seal a batch by size, so a +/// finalized "gold" snapshot promotes for a watchdog compare — independent of +/// the batch-open timer. +const TRANSFERS_TO_FORCE_BATCH_CLOSE: usize = 150; + +/// Poll budget for [`mine_until_finalized_advances`]: mine + recheck the +/// finalized snapshot up to this many times before timing out. With +/// [`PROMOTION_POLL_INTERVAL`] this bounds the wait at ~40s, generous against +/// CI scheduling jitter while never sleeping longer than the condition needs. +const PROMOTION_POLL_ATTEMPTS: usize = 40; + +/// Per-attempt pause in [`mine_until_finalized_advances`], giving the submitter +/// tick and the promoter room to run between mines. +const PROMOTION_POLL_INTERVAL: Duration = Duration::from_secs(1); + // ── Zone-math constants for the outage-matrix and recovery tests ───────── // // These derive from the sequencer's default config so a change to @@ -158,6 +174,9 @@ pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { ("recovery_after_stale_batches_test", |runtime| { Box::pin(run_recovery_after_stale_batches_test(runtime)) }), + ("setup_recovery_round_trip_test", |runtime| { + Box::pin(run_setup_recovery_round_trip_test(runtime)) + }), ("sequencer_outage_pre_danger_no_recovery_test", |runtime| { Box::pin(run_sequencer_outage_pre_danger_no_recovery_test(runtime)) }), @@ -385,8 +404,33 @@ async fn prepare_non_genesis_watchdog_state(runtime: &mut ManagedSequencer) -> S Ok(()) } +/// Mine L1 forward until a finalized snapshot is promoted at an inclusion block +/// strictly above `floor` (the value observed before the batch that should +/// promote), polling the DB instead of sleeping a fixed submitter-tick +/// interval. Each attempt mines a couple of blocks and pauses briefly, so the +/// submitter and promoter get to run; returns the new inclusion block, or times +/// out loudly. Robust against CI jitter — it waits exactly as long as promotion +/// takes, no more. +async fn mine_until_finalized_advances( + runtime: &ManagedSequencer, + floor: u64, +) -> ScenarioResult { + for _ in 0..PROMOTION_POLL_ATTEMPTS { + runtime.mine_l1_blocks(2).await?; + tokio::time::sleep(PROMOTION_POLL_INTERVAL).await; + let (inclusion_block, _) = runtime.finalized_snapshot_info()?; + if inclusion_block > floor { + return Ok(inclusion_block); + } + } + Err( + format!("timed out waiting for finalized snapshot to advance past inclusion_block {floor}") + .into(), + ) +} + /// Close the open batch, land it on L1, and wait for snapshot promotion so -/// `/finalized_state` reports `inclusion_block > 0`. +/// `/finalized_state` reports a new `inclusion_block`. async fn drive_finalized_gold_batch_for_watchdog( runtime: &ManagedSequencer, ws: &mut WsClient, @@ -394,9 +438,7 @@ async fn drive_finalized_gold_batch_for_watchdog( alice_l2: &mut WalletL2Client, alice_address: Address, ) -> ScenarioResult<()> { - const TRANSFERS_TO_FORCE_BATCH_CLOSE: usize = 150; - const SUBMITTER_TICK_WAIT: Duration = Duration::from_secs(7); - + let (floor_inclusion_block, _) = runtime.finalized_snapshot_info()?; let batches_before = runtime.count_batches()?; for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { alice_l2.transfer(alice_address, U256::from(1_u64)).await?; @@ -410,8 +452,7 @@ async fn drive_finalized_gold_batch_for_watchdog( .into()); } - tokio::time::sleep(SUBMITTER_TICK_WAIT).await; - runtime.mine_l1_blocks(3).await?; + mine_until_finalized_advances(runtime, floor_inclusion_block).await?; Ok(()) } @@ -1060,6 +1101,139 @@ async fn run_recovery_after_stale_batches_test( Ok(()) } +// ── Cockroach recovery: setup --recovery round-trip (PR5) ──────── +// +// The affirmative end-to-end test of cockroach recovery: build state, promote a +// real finalized snapshot (the checkpoint), wipe the DB, rebuild it from the +// checkpoint via `setup --recovery` (flush → fold → fill), boot `run`, and +// prove the recovered state is correct — the resumed sequencer accepts a new +// user-op at the *continuing* nonce (so the fold preserved the sender's nonce + +// balance), and the rebuilt tree is anchored at the resume nonce N' (validated +// structurally by the post-scenario `assert_schema_invariants`). +// +// `max_batch_open` is set to 5s (not the 2h default) so a batch seals + gets +// promoted via plain L1 mining — no 2h clock jump, so the L1 view stays fresh +// and no staleness/cascade fires. + +/// Mine + wait until a finalized snapshot is promoted past genesis (B > 0), +/// then capture it as a checkpoint. Times out loudly if no promotion lands. +async fn drive_promotion_and_capture( + runtime: &ManagedSequencer, +) -> ScenarioResult { + mine_until_finalized_advances(runtime, 0).await?; + runtime.capture_finalized_checkpoint() +} + +async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + runtime.set_max_batch_open_seconds(Some(5)); + runtime.restart().await?; + + let alice = TestSigner::from_default(1)?; + let bob = TestSigner::from_default(2)?; + let alice_address = alice.address(); + let bob_address = bob.address(); + let gas = fee_to_linear(DEFAULT_FRAME_FEE); + + let mut ws = runtime.ws(0).await?; + let alice_l1 = runtime.wallet_l1(alice.clone()).await?; + let mut alice_l2 = runtime.wallet_l2(alice.clone())?; + let mut replay = ReplayWalletApp::devnet(); + + // Build state: a deposit (direct) + a transfer (user-op). These land in the + // batch that will seal + promote into the checkpoint. + let deposit = U256::from(2_000_000_u64); + let transfer1 = U256::from(100_000_u64); + apply_safe_supported_deposit(runtime, &mut ws, &mut replay, &alice_l1, deposit).await?; + alice_l2.transfer(bob_address, transfer1).await?; + replay.apply(ws.expect_user_op_from(alice_address).await?)?; + let expected_alice = deposit - transfer1 - gas; + assert_eq!(replay.current_user_balance(alice_address), expected_alice); + assert_eq!(replay.current_user_balance(bob_address), transfer1); + assert_eq!(replay.current_user_nonce(alice_address), 1); + + // Drive the batch to seal + be accepted + promoted, and capture the + // resulting finalized snapshot as the recovery checkpoint. + let checkpoint = drive_promotion_and_capture(runtime).await?; + eprintln!( + "recovery checkpoint: B={} N={}", + checkpoint.checkpoint_block, checkpoint.resume_nonce + ); + + // Wipe the DB and rebuild it from the checkpoint via `setup --recovery`. + drop(ws); + runtime.stop().await?; + runtime.set_recovery_setup(Some(RecoverySetupParams { + checkpoint_block: checkpoint.checkpoint_block, + checkpoint_dump_dir: checkpoint.dir, + })); + runtime.reset_database()?; + // The recovery setup's flush + the post-recovery run boot both need L1 to + // keep advancing. + runtime.set_mine_l1_during_boot(true); + runtime.respawn().await?; + + // Recovery booted (the respawn above succeeded — `setup --recovery` rebuilt + // the DB and `run` started clean). Now prove the fold preserved Alice's + // logical state: a transfer at her *continuing* nonce (1) is accepted. The + // sequencer validates it against the recovered state S' — a lost nonce would + // be rejected (wrong nonce), a lost balance rejected (insufficient funds). + // So acceptance is the end-to-end proof that the checkpoint's balances + + // nonces were folded into the rebuilt DB. (The local `replay` can't verify + // S' directly — the wiped pre-recovery transfer isn't re-fed — so the + // sequencer's own acceptance is the authority here.) + let mut alice_l2_after = runtime.wallet_l2(alice)?; + alice_l2_after.set_next_nonce(1); + let post_transfer = U256::from(70_000_u64); + alice_l2_after.transfer(bob_address, post_transfer).await?; + + // Explicit recovery-correctness assertions, beyond the structural + // `assert_schema_invariants` (which checks `0..`-from-anchor contiguity): + // 1. the rebuilt tree is anchored at the checkpoint's resume nonce N' (I16); + // 2. recovery's resync did NOT spuriously freeze the frontier — the I15 + // content-identity false-positive against below-anchor collapsed history + // is otherwise a silent failure, invisible at the e2e level (the same + // silence the (C, H1] bug shipped behind). + assert_eq!( + runtime.batch_tree_anchor()?, + checkpoint.resume_nonce, + "the rebuilt tree must be anchored at the checkpoint resume nonce N'" + ); + assert_eq!( + runtime.canonical_divergence()?, + None, + "recovery must not falsely freeze the frontier (no canonical divergence)" + ); + + // ── Watchdog agreement on the rebuilt + resumed state ──────────────── + // + // The strongest end-to-end proof that `setup --recovery` reconstructed + // canonical state: drive a finalized gold batch on the *resumed* sequencer, + // then assert the watchdog's from-genesis CM replay is byte-identical to the + // finalized snapshot the sequencer now serves. The CM replays every L1 input + // from genesis — the pre-wipe batches (nonce < N') plus the post-recovery + // batches at the resume nonce N' — so agreement proves the fold-rebuilt state + // S' and the resumed submission both land exactly on the canonical chain + // state. The local `replay` can't check this (the wiped history is never + // re-fed), so the watchdog's independent CM is the authority. + let (floor_inclusion_block, _) = runtime.finalized_snapshot_info()?; + let batches_before = runtime.count_batches()?; + for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { + alice_l2_after + .transfer(alice_address, U256::from(1_u64)) + .await?; + } + let batches_after = runtime.count_batches()?; + assert!( + batches_after.sealed > batches_before.sealed, + "expected a sealed batch before the recovery watchdog compare: \ + before={batches_before:?} after={batches_after:?}" + ); + mine_until_finalized_advances(runtime, floor_inclusion_block).await?; + crate::watchdog_compare::run_watchdog_non_genesis_compare_test(runtime).await?; + + Ok(()) +} + // ── Sequencer outage, pre-danger zone ──────────────────────────── // // Sequencer stops with an open batch (deposit + transfer); L1 advances 500 @@ -2484,10 +2658,15 @@ async fn run_delayed_inclusion_cascades_on_restart_test( // faketime offset so the wall-clock fallback stays in sync with L1. runtime.advance_wall_and_mine(PAST_STALE).await?; - // Re-enable auto-mining before respawn: startup recovery's flush step - // submits a no-op at the stuck wallet-nonce slot and needs it mined - // to progress. With auto-mining off, the flusher would hang. + // Re-enable auto-mining AND mine L1 throughout the recovery boot. Startup + // recovery's WP2 flush submits a no-op at the stranded wallet-nonce slot + // and blocks until that slot is *safe* (`safe_nonce >= watermark + 1`). + // Auto-mining alone lands the no-op but mints no further blocks, so Anvil's + // `safe` tag never advances past it and the boot would hang; the boot miner + // supplies the steady block production a live chain would (see + // `ManagedSequencer::set_mine_l1_during_boot`). runtime.set_automine(true).await?; + runtime.set_mine_l1_during_boot(true); runtime.respawn().await?; @@ -3342,7 +3521,13 @@ async fn run_nonce_zero_recovery_invalidates_then_accepts_at_nonce_zero_test( runtime.drop_all_pending_txs().await?; runtime.advance_wall_and_mine(PAST_STALE).await?; + // Re-enable auto-mining AND mine L1 throughout the recovery boot: the WP2 + // flush submits a no-op at the stranded nonce-0 slot and waits for it to + // become safe. Auto-mining lands the no-op but mints no further blocks, so + // the boot miner supplies the steady block production needed to advance + // Anvil's `safe` tag past it (see `set_mine_l1_during_boot`). runtime.set_automine(true).await?; + runtime.set_mine_l1_during_boot(true); runtime.respawn().await?; @@ -3401,7 +3586,7 @@ async fn run_nonce_zero_recovery_invalidates_then_accepts_at_nonce_zero_test( // After confirmations land, the submitter's tick loop continues: // next iteration runs `refresh_recovery_metadata` → - // `populate_safe_accepted_batches_inner`, which appends the batch + // `populate_safe_accepted_batches`, which appends the batch // to `safe_accepted_batches` at its expected nonce (0, reused). tokio::time::sleep(Duration::from_secs(10)).await; diff --git a/tests/e2e/src/watchdog_compare.rs b/tests/e2e/src/watchdog_compare.rs index f376b22..11248a6 100644 --- a/tests/e2e/src/watchdog_compare.rs +++ b/tests/e2e/src/watchdog_compare.rs @@ -49,7 +49,7 @@ pub async fn run_watchdog_genesis_compare_test( .into()); } - // `sequencer-devnet` uses `WalletConfig::devnet()` (not `default()` / Sepolia). + // `wallet-sequencer-devnet` uses `WalletConfig::devnet()` (not `default()` / Sepolia). let expected_snapshot = wallet_snapshot::encode(&WalletApp::new(WalletConfig::devnet())); eprintln!("[watchdog-harness] step 1/6: wait for sequencer GET /finalized_state"); diff --git a/tests/fixtures/wallet_snapshot_empty.bin b/tests/fixtures/wallet_snapshot_empty.bin new file mode 100644 index 0000000..f7e8087 Binary files /dev/null and b/tests/fixtures/wallet_snapshot_empty.bin differ diff --git a/tests/fixtures/wallet_snapshot_empty.hex b/tests/fixtures/wallet_snapshot_empty.hex new file mode 100644 index 0000000..6d682e1 --- /dev/null +++ b/tests/fixtures/wallet_snapshot_empty.hex @@ -0,0 +1 @@ +aca6586a0cf05bd831f2501e7b4aea550da6562d1c7d4b196cb0c7b01d743fbc6116a902379c723816d5ff3fdd14e2a86fba77cbce6b3cd9c32b8ff3540000005400000000000000000000000000000000000000 diff --git a/tests/fixtures/wallet_snapshot_v1_empty.bin b/tests/fixtures/wallet_snapshot_v1_empty.bin deleted file mode 100644 index 047814d..0000000 Binary files a/tests/fixtures/wallet_snapshot_v1_empty.bin and /dev/null differ diff --git a/tests/fixtures/wallet_snapshot_v1_empty.hex b/tests/fixtures/wallet_snapshot_v1_empty.hex deleted file mode 100644 index 624ddd1..0000000 --- a/tests/fixtures/wallet_snapshot_v1_empty.hex +++ /dev/null @@ -1 +0,0 @@ -aca6586a0cf05bd831f2501e7b4aea550da6562d1c7d4b196cb0c7b01d743fbc6116a902379c723816d5ff3fdd14e2a86fba77cbce6b3cd9c32b8ff34c0000004c0000000000000000000000 diff --git a/tests/harness/src/lib.rs b/tests/harness/src/lib.rs index d7dcbc0..1e701d2 100644 --- a/tests/harness/src/lib.rs +++ b/tests/harness/src/lib.rs @@ -17,8 +17,8 @@ pub use replay::ReplayWalletApp; pub use rollups::{DEVNET_CHAIN_ID, DevnetRollupsStack}; pub use sequencer::{ BatchCounts, DEFAULT_DEVNET_SEQUENCER_BIN, DEFAULT_TEST_LOGS_DIR, ManagedSequencer, - ManagedSequencerConfig, RespawnAttemptOutcome, RespawnPolicy, default_devnet_sequencer_config, - devnet_sequencer_config_no_faketime, + ManagedSequencerConfig, RecoveryCheckpoint, RecoverySetupParams, RespawnAttemptOutcome, + RespawnPolicy, default_devnet_sequencer_config, devnet_sequencer_config_no_faketime, }; pub use wallet::{ TestSigner, WalletL1Client, WalletL2Client, address_from_signing_key, sign_user_op_hex, diff --git a/tests/harness/src/paths.rs b/tests/harness/src/paths.rs index e0de961..7fda203 100644 --- a/tests/harness/src/paths.rs +++ b/tests/harness/src/paths.rs @@ -39,15 +39,15 @@ pub fn devnet_machine_image_path() -> PathBuf { workspace_root().join(DEFAULT_DEVNET_MACHINE_IMAGE_PATH) } -const DEVNET_SEQUENCER_BIN: &str = "sequencer-devnet"; +const DEVNET_SEQUENCER_BIN: &str = "wallet-sequencer-devnet"; -/// Resolve the `sequencer-devnet` binary built for the current Cargo invocation. +/// Resolve the `wallet-sequencer-devnet` binary built for the current Cargo invocation. /// /// Prefers `CARGO_TARGET_DIR` (set by `cargo run` / `cargo test` in sandboxes and /// custom target dirs) over the workspace `target/debug/` tree, which may be stale /// when builds only run through Cargo with a redirected target directory. pub fn resolve_devnet_sequencer_bin() -> PathBuf { - if let Ok(path) = std::env::var("CARGO_BIN_EXE_SEQUENCER_DEVNET") { + if let Ok(path) = std::env::var("CARGO_BIN_EXE_WALLET_SEQUENCER_DEVNET") { let path = PathBuf::from(path); if path.exists() { return path; diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index 2890d5b..6bd3bfc 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -57,11 +57,18 @@ pub(crate) fn apply_ws_message( WsTxMessage::UserOp { sender, fee, data, .. } => { - app.execute_valid_user_op(&ValidUserOp { - sender: decode_address(sender.as_str()), - fee, - data: decode_hex_prefixed(data.as_str()), - })?; + // The WS feed does not carry the covering frame's safe_block + // yet (feed-protocol work, review F7/WP5), so the replayed + // app's safe-block clock lags the live one. Fine here: these + // replays assert balances/nonces, never the clock. + app.execute_valid_user_op( + &ValidUserOp { + sender: decode_address(sender.as_str()), + fee, + data: decode_hex_prefixed(data.as_str()), + }, + 0, + )?; } } Ok(()) diff --git a/tests/harness/src/sequencer.rs b/tests/harness/src/sequencer.rs index ae4402f..1b30baf 100644 --- a/tests/harness/src/sequencer.rs +++ b/tests/harness/src/sequencer.rs @@ -24,8 +24,21 @@ use crate::ws::WsClient; const DEFAULT_SEQUENCER_START_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(3); +/// Cadence at which the harness mines an L1 block during a recovery boot's +/// readiness wait when [`ManagedSequencer::set_mine_l1_during_boot`] is on. +/// Fast enough that Anvil's `safe` tag advances past the flushed nonce within +/// the readiness window; the exact value is a harness detail, not test-visible. +const BOOT_L1_MINE_INTERVAL: Duration = Duration::from_millis(200); +/// Readiness budget for a *recovery* boot (mining-during-boot enabled). The WP2 +/// mempool flush polls `get_transaction_count(Safe)` once per L1 block time +/// (`safe_poll_interval = seconds_per_block`, 12 s here), so it cannot resolve +/// the stranded slot in under one poll cycle — well past the 10 s normal-boot +/// budget. Allow several cycles of slack so a recovery boot has room to flush, +/// cascade, and come up. Recovery boots are legitimately slow (review R4 class +/// 10); this is the harness counterpart to that. +const RECOVERY_BOOT_START_TIMEOUT: Duration = Duration::from_secs(45); const DEFAULT_SEQUENCER_RUST_LOG: &str = "info"; -pub const DEFAULT_DEVNET_SEQUENCER_BIN: &str = "target/debug/sequencer-devnet"; +pub const DEFAULT_DEVNET_SEQUENCER_BIN: &str = "target/debug/wallet-sequencer-devnet"; pub const DEFAULT_TEST_LOGS_DIR: &str = "tests/e2e/results"; #[derive(Debug, Clone)] @@ -47,6 +60,31 @@ pub struct BatchCounts { pub invalidated: u64, } +/// Drive the next respawn's `setup` phase into `--recovery` mode against a +/// captured checkpoint. Set via [`ManagedSequencer::set_recovery_setup`] before +/// wiping the DB and respawning. +#[derive(Debug, Clone)] +pub struct RecoverySetupParams { + /// `B` — the checkpoint's L1 inclusion block (`--checkpoint-block`). + pub checkpoint_block: u64, + /// The captured checkpoint dump dir (`--checkpoint-dump-dir`). + pub checkpoint_dump_dir: PathBuf, +} + +/// A checkpoint captured from a running sequencer's finalized snapshot, ready to +/// drive `setup --recovery`. Returned by +/// [`ManagedSequencer::capture_finalized_checkpoint`]. +#[derive(Debug, Clone)] +pub struct RecoveryCheckpoint { + /// The copied dump dir (lives outside the DB/`dumps` reset scope, so it + /// survives [`ManagedSequencer::reset_database`]). + pub dir: PathBuf, + /// `B` — the finalized snapshot's L1 inclusion block. + pub checkpoint_block: u64, + /// `N` — the snapshot's recorded resume nonce (`info.toml next_batch_nonce`). + pub resume_nonce: u64, +} + /// Outcome of a single [`ManagedSequencer::respawn_and_watch`] attempt. #[derive(Debug)] pub enum RespawnAttemptOutcome { @@ -108,6 +146,20 @@ pub struct ManagedSequencer { /// [`Self::advance_wall_and_mine`]. Not touched by /// [`Self::set_faketime_offset`]. cumulative_offset_secs: u64, + /// When `true`, [`Self::respawn`] mines L1 blocks concurrently with the + /// post-boot readiness wait so a recovery boot's mempool flush can make + /// progress. See [`Self::set_mine_l1_during_boot`]. Persists across + /// respawns like the other overrides. + mine_l1_during_boot: bool, + /// When `Some`, [`Self::respawn`] runs the `setup` phase in `--recovery` + /// mode against this checkpoint (and mines L1 concurrently with setup so + /// its flush can settle). See [`Self::set_recovery_setup`]. + recovery_setup: Option, + /// When `Some(secs)`, passes `--max-batch-open-seconds` to the `run` phase + /// so the inclusion lane seals the open batch promptly (the default is 2h). + /// Used by tests that need a batch to close + be promoted. Persists across + /// respawns. See [`Self::set_max_batch_open_seconds`]. + max_batch_open_seconds: Option, } pub fn default_devnet_sequencer_config(log_prefix: impl Into) -> ManagedSequencerConfig { @@ -169,6 +221,13 @@ impl ManagedSequencer { None, libfaketime_path.as_deref(), faketime_rc_path.as_deref(), + // First boot has no stranded nonce to flush, so it never needs L1 + // to advance during readiness. + false, + // First boot is always genesis setup, never recovery. + None, + // Default batch-open deadline on first boot. + None, ) .await?; @@ -188,6 +247,9 @@ impl ManagedSequencer { faketime_rc_path, libfaketime_path, cumulative_offset_secs: 0, + mine_l1_during_boot: false, + recovery_setup: None, + max_batch_open_seconds: None, }) } @@ -200,18 +262,48 @@ impl ManagedSequencer { self.l1_endpoint_override = l1_endpoint; } - /// Override the `--chain-id` argument the sequencer is spawned with on - /// the next [`Self::respawn`]. When `None`, defaults to the devnet - /// chain id (matches Anvil). + /// Override the `--chain-id` argument passed to the `setup` phase on the + /// next [`Self::respawn`]. When `None`, defaults to the devnet chain id + /// (matches Anvil). /// - /// Used by chain-id mismatch tests to inject a mismatched chain id and assert - /// that bootstrap refuses before silently pinning a wrong deployment - /// identity. Does not affect - /// the currently-running sequencer process. + /// Used by chain-id mismatch tests to inject a mismatched chain id and + /// assert that bootstrap refuses before silently pinning a wrong identity. + /// NOTE (post setup/run split): `--chain-id` now flows only to `setup`, + /// which is idempotent and early-returns once the `setup_complete` marker + /// exists — so the override only takes effect on a fresh or + /// [`Self::reset_database`]-d data dir. On an already-set-up DB, `setup` + /// is a no-op and `run` validates against the *pinned* chain id, so the + /// override is silently inert. Does not affect a running process. pub fn set_chain_id_override(&mut self, chain_id: Option) { self.chain_id_override = chain_id; } + /// Make [`Self::respawn`] (this one and subsequent ones, until cleared) + /// mine L1 blocks concurrently with the post-boot readiness wait. + /// + /// Why this exists: a *recovery* boot whose persisted state names a wallet + /// nonce that L1 never accepted — e.g. a batch-submission tx was dropped + /// from the mempool ([`Self::drop_all_pending_txs`]) — runs the WP2 mempool + /// flush during startup. The flush submits a no-op at the stranded nonce + /// slot and blocks until that slot becomes *safe* (`safe_nonce >= + /// watermark + 1`, review R1a). Re-enabling auto-mining is not enough: + /// auto-mining lands the no-op tx but mints no *further* blocks, so Anvil's + /// `safe` tag never advances past it and the flush waits until the boot + /// hits the readiness timeout. In production the chain keeps producing + /// blocks throughout a slow recovery boot; an on-demand-mining devnet + /// produces none. This makes the harness mine a steady trickle for the + /// boot window (dropped the instant readiness is reached), reproducing that + /// drift so the flush — and the cascade behind it — can complete on the + /// first respawn. + /// + /// Only the tests that deliberately strand a submitted nonce before a + /// recovery respawn need this. Leave it off otherwise: every other boot + /// reaches readiness without new blocks, and mining empty blocks during an + /// unrelated boot would perturb L1-block-sensitive assertions. + pub fn set_mine_l1_during_boot(&mut self, enabled: bool) { + self.mine_l1_during_boot = enabled; + } + /// Write a faketime offset to the rc file. Effective **immediately** for /// the running sequencer (if any) and persists across respawns. The /// libfaketime library re-reads the file on every time call (we pass @@ -239,17 +331,21 @@ impl ManagedSequencer { Ok(()) } - /// Delete the sequencer DB file (and its `-wal` / `-shm` siblings), - /// simulating a brand-new install — no pinned identity, no batches, no - /// safe-input rows. Call while the sequencer is stopped. + /// Delete the sequencer DB file (and its `-wal` / `-shm` siblings) AND the + /// `dumps/` subtree, simulating a brand-new install — no pinned identity, + /// no batches, no safe-input rows, no genesis snapshot. Call while the + /// sequencer is stopped. (Clearing the DB but not `dumps/` would leave a + /// stale `genesis-*` dir; the next `setup` mints a fresh one and the old + /// one lingers until run's orphan sweep — harmless, but not "brand-new".) /// /// Distinct from "clear only the identity row": that would leave the DB /// holding `batches` / `safe_inputs` / `l1_safe_head` rows from prior /// boots, which `IdentityError::OrphanedState` then refuses on the next - /// boot. The right "first boot" simulation is to start from an empty DB. + /// `setup`. The right "first boot" simulation is to start from an empty DB. /// /// Used by the no-cache-bootstrap and live-RPC chain-id-mismatch tests: - /// both want to exercise the no-cached-identity bootstrap path. + /// both want to exercise the no-cached-identity bootstrap path (next boot + /// re-runs `setup`). pub fn reset_database(&self) -> HarnessResult<()> { let db_path = self.data_dir_path.join("sequencer.db"); for suffix in ["", "-wal", "-shm"] { @@ -260,9 +356,89 @@ impl ManagedSequencer { Err(err) => return Err(io_other(format!("reset DB ({path:?}): {err}")).into()), } } + let dumps_dir = self.data_dir_path.join("dumps"); + match fs::remove_dir_all(dumps_dir.as_path()) { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(io_other(format!("reset dumps ({dumps_dir:?}): {err}")).into()), + } Ok(()) } + /// Drive the next [`Self::respawn`]'s `setup` phase into `--recovery` against + /// `params` (and mine L1 concurrently with setup so its flush can settle). + /// Set this after capturing a checkpoint and wiping the DB. `None` restores + /// the normal (genesis) setup. Persists across respawns until cleared. + pub fn set_recovery_setup(&mut self, params: Option) { + self.recovery_setup = params; + } + + /// Pass `--max-batch-open-seconds` to the `run` phase on the next (and + /// subsequent) respawns so the inclusion lane seals the open batch after + /// `secs` instead of the 2h default. `None` restores the default. Lets a + /// test drive a batch to close + promotion without a 2h clock advance. + pub fn set_max_batch_open_seconds(&mut self, secs: Option) { + self.max_batch_open_seconds = secs; + } + + /// Read the finalized snapshot's `(inclusion_block B, resume_nonce N)`. `B` + /// is `0` for the genesis snapshot and `> 0` once a real batch has been + /// promoted — poll this (mining L1 in between) to wait for a recoverable + /// checkpoint. Read-only. + pub fn finalized_snapshot_info(&self) -> HarnessResult<(u64, u64)> { + let db_path = self.data_dir_path.join("sequencer.db"); + let conn = rusqlite::Connection::open_with_flags( + db_path.as_path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|err| io_other(format!("open DB read-only: {err}")))?; + let (prefix, inclusion_block): (String, i64) = conn + .query_row( + "SELECT d.prefix, f.inclusion_block FROM finalized_snapshot f \ + JOIN dumps d ON d.id = f.dump_id WHERE f.singleton_id = 0", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|err| io_other(format!("read finalized_snapshot: {err}")))?; + let resume_nonce = read_info_next_batch_nonce(Path::new(&prefix))?; + Ok((inclusion_block as u64, resume_nonce)) + } + + /// Copy the current finalized snapshot dump to `/checkpoint` (which + /// survives [`Self::reset_database`], since that only clears `sequencer.db*` + /// and `dumps/`), returning the captured checkpoint. Call after a batch has + /// been promoted (`finalized_snapshot_info().0 > 0`). + pub fn capture_finalized_checkpoint(&self) -> HarnessResult { + let db_path = self.data_dir_path.join("sequencer.db"); + let conn = rusqlite::Connection::open_with_flags( + db_path.as_path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|err| io_other(format!("open DB read-only: {err}")))?; + let (prefix, inclusion_block): (String, i64) = conn + .query_row( + "SELECT d.prefix, f.inclusion_block FROM finalized_snapshot f \ + JOIN dumps d ON d.id = f.dump_id WHERE f.singleton_id = 0", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|err| io_other(format!("read finalized_snapshot: {err}")))?; + let src = PathBuf::from(prefix); + let dst = self.data_dir_path.join("checkpoint"); + if dst.exists() { + fs::remove_dir_all(dst.as_path()) + .map_err(|err| io_other(format!("clear prior checkpoint ({dst:?}): {err}")))?; + } + copy_dir_recursive(src.as_path(), dst.as_path()) + .map_err(|err| io_other(format!("copy checkpoint {src:?} -> {dst:?}: {err}")))?; + let resume_nonce = read_info_next_batch_nonce(dst.as_path())?; + Ok(RecoveryCheckpoint { + dir: dst, + checkpoint_block: inclusion_block as u64, + resume_nonce, + }) + } + /// Rewrite the L1 safe-head observation to "unknown", simulating a DB /// that has never successfully synced from L1. Call while the sequencer /// is stopped. @@ -290,7 +466,7 @@ impl ManagedSequencer { /// /// Used by the nonce-0 recovery test to confirm a recovery batch (which reuses nonce 0) /// actually lands and gets accepted on L1 — proving the - /// `populate_safe_accepted_batches_inner` cursor handles + /// `populate_safe_accepted_batches` cursor handles /// reused-nonce-after-cascade correctly. pub fn count_safe_accepted_batches(&self) -> HarnessResult<(u64, Option)> { let db_path = self.data_dir_path.join("sequencer.db"); @@ -313,6 +489,48 @@ impl ManagedSequencer { Ok((count as u64, min_nonce.map(|n| n as u64))) } + /// The batch-tree anchor nonce, read from the run DB read-only: `N'` after + /// cockroach recovery, `0` for a genesis deployment. Lets a recovery e2e + /// assert the rebuilt tree is rooted at the checkpoint's resume nonce (I16). + pub fn batch_tree_anchor(&self) -> HarnessResult { + let db_path = self.data_dir_path.join("sequencer.db"); + let conn = rusqlite::Connection::open_with_flags( + db_path.as_path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|err| io_other(format!("open DB read-only: {err}")))?; + let anchor: i64 = conn + .query_row( + "SELECT nonce FROM batch_tree_anchor WHERE singleton_id = 0", + [], + |row| row.get(0), + ) + .map_err(|err| io_other(format!("read batch_tree_anchor: {err}")))?; + Ok(anchor as u64) + } + + /// The canonical-divergence marker (review R2/I15) from the run DB, or `None` + /// when the frontier is healthy. A recovery/resync e2e asserts this is `None` + /// to prove the content-identity check did NOT spuriously freeze the frontier + /// (e.g. a false-positive against below-anchor collapsed history) — otherwise + /// that silent-failure class is invisible at e2e level. + pub fn canonical_divergence(&self) -> HarnessResult> { + use rusqlite::OptionalExtension; + let db_path = self.data_dir_path.join("sequencer.db"); + let conn = rusqlite::Connection::open_with_flags( + db_path.as_path(), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|err| io_other(format!("open DB read-only: {err}")))?; + conn.query_row( + "SELECT kind FROM canonical_divergence WHERE singleton_id = 0", + [], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|err| io_other(format!("read canonical_divergence: {err}")).into()) + } + /// Snapshot of the `batches` table: `(total, sealed, invalidated)`. /// Reads the DB file read-only; safe to call while the sequencer is /// running. Useful for asserting that batch closure happened during a @@ -377,6 +595,15 @@ impl ManagedSequencer { ) .map_err(|err| io_other(format!("open DB read-only: {err}")))?; + // The batch-tree anchor: the nonce the single parentless root carries + // (0 for a genesis deployment, N' for a cockroach-recovered one). The + // root/contiguity invariants below are stated relative to it, so a + // recovered tree (rooted at N') passes the same checks a genesis tree + // (rooted at 0) does. Mirrors `trg_enforce_nonce_contiguity`. Reuses the + // `batch_tree_anchor()` accessor (its own short-lived read-only handle) + // rather than re-issuing the query against `conn`. + let anchor = self.batch_tree_anchor()? as i64; + // 1. At most one valid open batch. let open_count: i64 = conn .query_row("SELECT COUNT(*) FROM valid_open_batch", [], |row| { @@ -387,6 +614,20 @@ impl ManagedSequencer { panic!("schema invariant: more than one valid Tip ({open_count} rows)"); } + // 1b. At most one *valid* parentless root (the anchored root). + let valid_root_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM valid_batches WHERE parent_batch_index IS NULL", + [], + |row| row.get(0), + ) + .map_err(|err| io_other(format!("count valid roots: {err}")))?; + if valid_root_count > 1 { + panic!( + "schema invariant: more than one valid parentless root ({valid_root_count} rows)" + ); + } + // 2. Nonce contiguity via parent. let mut stmt = conn .prepare( @@ -404,9 +645,9 @@ impl ManagedSequencer { for (bi, parent, nonce, parent_nonce) in &rows { match (parent, parent_nonce) { (None, _) => { - if *nonce != 0 { + if *nonce != anchor { panic!( - "schema invariant: batch {bi} has NULL parent but nonce {nonce} (expected 0)" + "schema invariant: batch {bi} has NULL parent but nonce {nonce} (expected anchor {anchor})" ); } } @@ -444,8 +685,10 @@ impl ManagedSequencer { } } for (i, &n) in valid_nonces.iter().enumerate() { - if n != i as i64 { - panic!("schema invariant: valid nonces not contiguous: {valid_nonces:?}"); + if n != anchor + i as i64 { + panic!( + "schema invariant: valid nonces not contiguous from anchor {anchor}: {valid_nonces:?}" + ); } } @@ -711,6 +954,9 @@ impl ManagedSequencer { self.chain_id_override, self.libfaketime_path.as_deref(), self.faketime_rc_path.as_deref(), + self.mine_l1_during_boot, + self.recovery_setup.as_ref(), + self.max_batch_open_seconds, ) .await?; self.child = child; @@ -790,6 +1036,42 @@ struct SpawnedSequencerProcess { log_path: PathBuf, } +/// Read `next_batch_nonce` out of a dump dir's `info.toml` (the resume nonce +/// `N`). The file is the simple `key = value` form the sequencer writes. +fn read_info_next_batch_nonce(dump_dir: &Path) -> HarnessResult { + let info_path = dump_dir.join("info.toml"); + let content = fs::read_to_string(info_path.as_path()) + .map_err(|err| io_other(format!("read {info_path:?}: {err}")))?; + for line in content.lines() { + if let Some((key, value)) = line.split_once(" = ") + && key.trim() == "next_batch_nonce" + { + return value + .trim() + .parse::() + .map_err(|err| io_other(format!("parse next_batch_nonce: {err}")).into()); + } + } + Err(io_other(format!("info.toml missing next_batch_nonce: {info_path:?}")).into()) +} + +/// Recursively copy a directory tree (used to stash a checkpoint dump where +/// [`ManagedSequencer::reset_database`] won't wipe it). +fn copy_dir_recursive(src: &Path, dst: &Path) -> io::Result<()> { + fs::create_dir_all(dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let from = entry.path(); + let to = dst.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_dir_recursive(&from, &to)?; + } else { + fs::copy(&from, &to)?; + } + } + Ok(()) +} + #[allow(clippy::too_many_arguments)] async fn spawn_sequencer_process( sequencer_bin: &Path, @@ -801,36 +1083,72 @@ async fn spawn_sequencer_process( chain_id_override: Option, libfaketime_path: Option<&Path>, faketime_rc_path: Option<&Path>, + mine_l1_during_boot: bool, + recovery: Option<&RecoverySetupParams>, + max_batch_open_seconds: Option, ) -> HarnessResult { let (endpoint, http_addr) = build_local_endpoint()?; let log_path = timestamped_log_path(logs_dir, log_prefix); - let stdout_log = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(log_path.as_path())?; - let stderr_log = stdout_log.try_clone()?; - - let batch_submitter_key = default_private_keys().first().cloned().unwrap_or_else(|| { - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80".to_string() - }); + + // Dedicated batch submitter = anvil account 9 (NOT the deployer/funder, + // account 0). `setup`'s detection gate refuses when the + // submitter's wallet nonce is unsettled; the deployer's deploy-tx tail is + // not safe at setup time, so reusing it false-positives. Account 9 starts + // at nonce 0. Must match `DEVNET_SEQUENCER_ADDRESS` (app-core); a debug + // assert below pins that. Account 9 is the highest standard funded anvil + // account, so it stays clear of the e2e wallets (0–3) and the bench + // workload (indices 0..concurrency). + const DEVNET_SUBMITTER_ACCOUNT_INDEX: usize = 9; + let batch_submitter_key = default_private_keys() + .get(DEVNET_SUBMITTER_ACCOUNT_INDEX) + .cloned() + .unwrap_or_else(|| { + "0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6".to_string() + }); + // `setup` takes the submitter *address* (it never signs); `run` takes the + // key. Derive the address from the key so the harness configures both + // consistently. + let submitter_address = TestSigner::from_private_key_hex(&batch_submitter_key)?.address(); + // `assert_eq!`, not `debug_assert_eq!`: benches build the harness `--release`, + // where a debug-assert is a no-op — drift between app-core and the harness + // submitter must fail fast in every build. Cost is one comparison per spawn. + assert_eq!( + submitter_address, + app_core::application::DEVNET_SEQUENCER_ADDRESS, + "harness submitter (anvil account {DEVNET_SUBMITTER_ACCOUNT_INDEX}) must equal \ + DEVNET_SEQUENCER_ADDRESS — keep app-core and the harness in sync" + ); + let batch_submitter_address = submitter_address.to_string(); let eth_rpc_url = l1_endpoint_override.unwrap_or_else(|| rollups.l1_endpoint()); - // Set up libfaketime via env vars (not the `faketime` wrapper binary). - // The wrapper sets the FAKETIME env var, which has priority over - // FAKETIME_TIMESTAMP_FILE — bypassing it lets the file-based mechanism - // work. The file's contents are re-read on every `SystemTime::now()` / - // `Instant::now()` call thanks to FAKETIME_NO_CACHE=1, so tests can - // shift the clock dynamically during a run. - let mut cmd = Command::new(path_as_str(sequencer_bin)?); + let chain_id = chain_id_override.unwrap_or(DEVNET_CHAIN_ID); + let bin = path_as_str(sequencer_bin)?.to_owned(); + + // libfaketime is applied via env vars (not the `faketime` wrapper binary), + // which the file-based FAKETIME_TIMESTAMP_FILE mechanism reads on every + // time call (FAKETIME_NO_CACHE=1) so tests can shift the clock at runtime. + let open_log = |append: bool| -> HarnessResult { + Ok(OpenOptions::new() + .create(true) + .write(true) + .truncate(!append) + .append(append) + .open(log_path.as_path())?) + }; + + // Phase A — `setup` (run once; idempotent on re-spawns, so it is a cheap + // no-op once the DB is set up, and needs L1 only on the first run / after + // a DB reset). Run to completion as a blocking step; a bootstrap refusal + // (e.g. a chain-id mismatch) surfaces here as a non-zero exit, just as it + // surfaced from the monolithic boot before the split. + let mut setup_cmd = Command::new(&bin); if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { - apply_faketime_env(&mut cmd, lib, rc)?; + apply_faketime_env(&mut setup_cmd, lib, rc)?; } - - let chain_id = chain_id_override.unwrap_or(DEVNET_CHAIN_ID); - let mut child = cmd - .arg("--http-addr") - .arg(http_addr) + let setup_log = open_log(false)?; + let setup_log_err = setup_log.try_clone()?; + setup_cmd + .arg("setup") .arg("--data-dir") .arg(path_as_str(data_dir)?) .arg("--eth-rpc-url") @@ -839,25 +1157,152 @@ async fn spawn_sequencer_process( .arg(chain_id.to_string()) .arg("--app-address") .arg(rollups.app_address().to_string()) - .arg("--batch-submitter-private-key") - .arg(&batch_submitter_key) + .arg("--batch-submitter-address") + .arg(&batch_submitter_address); + if let Some(recovery) = recovery { + // Recovery setup rebuilds the wiped DB from a checkpoint: it signs (the + // flush), so it takes the key, plus the checkpoint block + dump dir. + setup_cmd + .arg("--recovery") + .arg("--checkpoint-block") + .arg(recovery.checkpoint_block.to_string()) + .arg("--checkpoint-dump-dir") + .arg(path_as_str(&recovery.checkpoint_dump_dir)?) + .arg("--batch-submitter-private-key") + .arg(&batch_submitter_key); + } + setup_cmd .env("RUST_LOG", DEFAULT_SEQUENCER_RUST_LOG) - .stdout(Stdio::from(stdout_log)) - .stderr(Stdio::from(stderr_log)) + .stdout(Stdio::from(setup_log)) + .stderr(Stdio::from(setup_log_err)); + let mut setup_child = setup_cmd .spawn() - .map_err(|err| { - io_other(format!( - "failed to spawn sequencer binary '{}': {err}", - sequencer_bin.display() - )) - })?; + .map_err(|err| io_other(format!("failed to spawn `setup` for '{bin}': {err}")))?; + // Bound the setup phase so a stalled/lagging RPC during the initial sync + // can't hang the harness indefinitely. Recovery setup additionally flushes + // the wallet nonce, which can block until a stranded slot becomes safe — so + // mine L1 concurrently (nothing else does on an on-demand devnet) and give + // it the generous recovery-boot budget. Genesis setup never flushes. + let setup_status = if recovery.is_some() { + let deadline = std::time::Instant::now() + RECOVERY_BOOT_START_TIMEOUT; + loop { + if let Some(status) = setup_child + .try_wait() + .map_err(|err| io_other(format!("poll recovery `setup`: {err}")))? + { + break status; + } + if std::time::Instant::now() >= deadline { + let _ = setup_child.start_kill(); + let _ = setup_child.wait().await; + return Err(io_other(format!( + "recovery `setup` timed out after {:?} (see {})", + RECOVERY_BOOT_START_TIMEOUT, + log_path.display() + )) + .into()); + } + let _ = rollups.mine_l1_blocks(1).await; + tokio::time::sleep(BOOT_L1_MINE_INTERVAL).await; + } + } else { + match tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, setup_child.wait()).await { + Ok(status) => status?, + Err(_) => { + let _ = setup_child.start_kill(); + let _ = setup_child.wait().await; + return Err(io_other(format!( + "sequencer `setup` timed out after {:?} (see {})", + DEFAULT_SEQUENCER_START_TIMEOUT, + log_path.display() + )) + .into()); + } + } + }; + if !setup_status.success() { + return Err(io_other(format!( + "sequencer `setup` failed: status={setup_status} (see {})", + log_path.display() + )) + .into()); + } - wait_for_http_readiness( - endpoint.as_str(), - &mut child, - DEFAULT_SEQUENCER_START_TIMEOUT, - ) - .await?; + // Phase B — `run` (re-spawned on every restart; reads identity from the + // setup DB). + let mut run_cmd = Command::new(&bin); + if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { + apply_faketime_env(&mut run_cmd, lib, rc)?; + } + let run_log = open_log(true)?; + let run_log_err = run_log.try_clone()?; + // `kill_on_drop`: a `ManagedSequencer` normally reaps its child via + // `shutdown()` / `stop()`, but an error path that drops the struct without + // either (e.g. a benchmark returning early on a misconfigured run) would + // otherwise leave the child running while the owning `TempDir` deletes the + // data dir — the child's next DB open then fails `ENOENT` and it exits with + // a confusing `unable to open database file` that masks the real failure. + // `child` is declared before `_data_dir`, so on drop the SIGKILL lands + // before the dir is removed. + run_cmd + .kill_on_drop(true) + .arg("run") + .arg("--http-addr") + .arg(http_addr) + .arg("--data-dir") + .arg(path_as_str(data_dir)?) + .arg("--eth-rpc-url") + .arg(eth_rpc_url) + .arg("--batch-submitter-private-key") + .arg(&batch_submitter_key); + if let Some(secs) = max_batch_open_seconds { + run_cmd + .arg("--max-batch-open-seconds") + .arg(secs.to_string()); + } + run_cmd + .env("RUST_LOG", DEFAULT_SEQUENCER_RUST_LOG) + .stdout(Stdio::from(run_log)) + .stderr(Stdio::from(run_log_err)); + let mut child = run_cmd.spawn().map_err(|err| { + io_other(format!( + "failed to spawn sequencer binary '{}': {err}", + sequencer_bin.display() + )) + })?; + + if mine_l1_during_boot { + // A recovery boot blocks in the WP2 mempool flush until the stranded + // wallet-nonce slot becomes safe, which needs L1 to keep producing + // blocks. On an on-demand-mining devnet nothing produces them, so mine + // a trickle concurrently with the readiness wait — mirroring a live + // chain that advances during a slow boot. The miner is dropped the + // instant readiness resolves, so L1 only advances for the boot window. + let miner = async { + loop { + tokio::time::sleep(BOOT_L1_MINE_INTERVAL).await; + // Transient RPC hiccups mid-boot are non-fatal — the next tick + // retries. A genuine mining failure manifests as the boot + // timing out, reported by the readiness arm. + let _ = rollups.mine_l1_blocks(1).await; + } + }; + tokio::select! { + res = wait_for_http_readiness( + endpoint.as_str(), + &mut child, + RECOVERY_BOOT_START_TIMEOUT, + ) => res?, + _ = miner => unreachable!("boot miner loops forever; only readiness resolves"), + } + } else { + wait_for_http_readiness( + endpoint.as_str(), + &mut child, + DEFAULT_SEQUENCER_START_TIMEOUT, + ) + .await?; + } Ok(SpawnedSequencerProcess { child, diff --git a/tests/harness/src/wallet.rs b/tests/harness/src/wallet.rs index f713c58..0491896 100644 --- a/tests/harness/src/wallet.rs +++ b/tests/harness/src/wallet.rs @@ -243,6 +243,14 @@ impl WalletL2Client { }) } + /// Override the next user-op nonce this client will sign. A fresh client + /// starts at 0; after a state-preserving recovery (where the sender's nonce + /// carries over) a test sets this to the sender's continuing nonce so its + /// post-recovery ops are accepted. + pub fn set_next_nonce(&mut self, nonce: u32) { + self.next_nonce = nonce; + } + pub async fn transfer(&mut self, to: Address, amount: U256) -> HarnessResult { self.submit_method(Method::Transfer(Transfer { amount, to })) .await diff --git a/watchdog/tests/run.lua b/watchdog/tests/run.lua index c462d8f..d2fadb6 100644 --- a/watchdog/tests/run.lua +++ b/watchdog/tests/run.lua @@ -76,7 +76,7 @@ local function fail_lookup(fail_ranges) end local function load_wallet_snapshot_hex_fixture() - local path = "tests/fixtures/wallet_snapshot_v1_empty.hex" + local path = "tests/fixtures/wallet_snapshot_empty.hex" local file, err = io.open(path, "rb") if not file then error("open " .. path .. ": " .. tostring(err)) @@ -93,7 +93,7 @@ end test("wallet SSZ golden fixture loads for cross-stack parity", function() local bytes = load_wallet_snapshot_hex_fixture() assert(#bytes > 0, "golden fixture must not be empty") - -- Fixed prefix from WalletSnapshotV1 default config (see wallet_snapshot.rs tests). + -- Fixed prefix from WalletSnapshot default config (see wallet_snapshot.rs tests). assert_eq(bytes:byte(1), 0xac) assert_eq(bytes:byte(2), 0xa6) end)