From 12e6382c5932836e1b6d6d9b1e0fe23b5e6089e9 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 13:29:52 +0200 Subject: [PATCH 01/14] shadowfork-image-build: the geth half of the post-Gloas benchmark images A fresh mainnet snapshot advanced by one pre-run (EIP-8282 contracts on Osaka, Amsterdam at head + 3 s, 24 KiB + 64 KiB receivers at 100k per variant), its gas limit walked down to 200M, the fill flushed out of geth's journal and compacted, then the geth image and the replay bundle uploaded to ethpandaops-shadowfork-images. The other clients replay the bundle separately and latest is written once all five are up (shadowfork/README.md). Clean image: geth's path scheme journaled 1,354 blocks of the last fill on a clean stop, so the newest contracts were read from memory. The ramp-down runs with --cache.gc=0 (0-byte write buffer) and the step fails unless the journal holds only the last 128 diff layers; then geth db compact. Verified on a dev chain (layers 176 -> 128); rampdown.py built 256 Amsterdam blocks (V5, slot numbers, 11.5M -> 10M) whose bundle replayed on a second node to the same head. Runs on the synctest Disk4TB runners (the gasbenchmark hosts hold the jochemnet CI baselines); uses the repo's existing S3 credentials and probes them before downloading anything. --- .github/workflows/shadowfork-image-build.yaml | 145 ++++++++++++++++++ shadowfork/.gitignore | 1 + shadowfork/README.md | 71 +++++++++ shadowfork/blockhash.py | 17 ++ shadowfork/finalize.sh | 50 ++++++ shadowfork/inplace-schelk | 29 ++++ shadowfork/lib.sh | 39 +++++ shadowfork/pfetch.sh | 48 ++++++ shadowfork/prepare.sh | 47 ++++++ shadowfork/prerun-config.sh | 58 +++++++ shadowfork/publish.sh | 29 ++++ shadowfork/rampdown.py | 78 ++++++++++ shadowfork/verify.sh | 28 ++++ 13 files changed, 640 insertions(+) create mode 100644 .github/workflows/shadowfork-image-build.yaml create mode 100644 shadowfork/.gitignore create mode 100644 shadowfork/README.md create mode 100755 shadowfork/blockhash.py create mode 100755 shadowfork/finalize.sh create mode 100755 shadowfork/inplace-schelk create mode 100755 shadowfork/lib.sh create mode 100755 shadowfork/pfetch.sh create mode 100755 shadowfork/prepare.sh create mode 100755 shadowfork/prerun-config.sh create mode 100755 shadowfork/publish.sh create mode 100755 shadowfork/rampdown.py create mode 100755 shadowfork/verify.sh diff --git a/.github/workflows/shadowfork-image-build.yaml b/.github/workflows/shadowfork-image-build.yaml new file mode 100644 index 0000000..c752bc0 --- /dev/null +++ b/.github/workflows/shadowfork-image-build.yaml @@ -0,0 +1,145 @@ +name: shadowfork-image-build + +# Builds the geth half of a post-Gloas benchmark image: a fresh mainnet snapshot advanced by one +# pre-run (EIP-8282 contracts, Amsterdam, 24 KiB + 64 KiB receivers), its gas limit walked down, +# the fill flushed out of geth's journal and compacted. Uploads the geth image and the replay +# bundle; the other clients replay the bundle afterwards (shadowfork/README.md). `latest` is +# written only once every client is up. +run-name: "shadowfork-image-build · anchor=${{ inputs.anchor }} · receivers=${{ inputs.receivers }} · gas=${{ inputs.gas-target }}" + +on: + workflow_dispatch: + inputs: + anchor: + description: "Mainnet block of the snapshots.ethpandaops.io geth snapshot to start from (or latest)" + default: "latest" + receivers: + description: "Receivers per variant, at each code size (100000 covers benchmarks up to ~300 Mgas, 321000 needs 1 Ggas)" + default: "100000" + gas-target: + description: "Gas limit of the published head" + default: "200000000" + eest-repo: + description: "execution-specs repo for the fill" + default: "https://github.com/qu0b/execution-specs.git" + eest-ref: + description: "execution-specs ref: needs the combined-sizes setup test and skylenet's fixture spill" + default: "qu0b/bench-images-v2" + geth-image: + description: "geth release that fills and opens the image (the one the devnets run)" + default: "ethereum/client-go:v1.17.7" + benchmarkoor-image: + description: "benchmarkoor image" + default: "ghcr.io/ethpandaops/benchmarkoor:master" + runner: + description: "Runner (JSON). Needs ~2 TB free for a mainnet geth datadir plus the fill" + default: '{"group": "synctest", "labels": "Disk4TB"}' + publish: + description: "Upload the image and bundle (false: build and verify locally only)" + type: boolean + default: true + +concurrency: + group: shadowfork-image-build + cancel-in-progress: false + +jobs: + geth: + runs-on: ${{ fromJSON(inputs.runner) }} + timeout-minutes: 2880 + env: + GETH_IMAGE: ${{ inputs.geth-image }} + S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v5 + + - name: Environment + run: | + echo "SF_WORK=${{ runner.temp }}/shadowfork" >> "$GITHUB_ENV" + for t in docker jq zstd curl python3; do command -v $t >/dev/null || { echo "::error::$t missing"; exit 1; }; done + command -v rclone >/dev/null || curl -fsSL https://rclone.org/install.sh | sudo bash + df -h "${{ runner.temp }}" + + - name: Bucket access + if: inputs.publish + run: | + source shadowfork/lib.sh + p="build-test/probe-${{ github.run_id }}" + if ! { echo probe | r2 rcat "r2:$BUCKET/$p" && r2 deletefile "r2:$BUCKET/$p"; }; then + echo "::error::the repo's S3 credentials cannot write $BUCKET"; exit 1 + fi + + - name: Snapshot and genesis + run: shadowfork/prepare.sh "${{ inputs.anchor }}" + + - name: Pre-run config + id: cfg + env: + RECEIVERS: ${{ inputs.receivers }} + EEST_REPO: ${{ inputs.eest-repo }} + EEST_REF: ${{ inputs.eest-ref }} + run: | + source shadowfork/lib.sh; source "$SF_WORK/anchor.env" + # The pre-run advances the datadir in place: schelk's CLI over a bind mount, no baseline. + { echo "SCHELK_DIR=$DATADIR"; echo "SCHELK_STATE=$SF_WORK/.schelk/state.json" + echo "BENCHMARKOOR_SCHELK_BIN=$PWD/shadowfork/inplace-schelk"; echo "TMPDIR=$SF_WORK/tmp"; } >> "$GITHUB_ENV" + mkdir -p "$SF_WORK/tmp" + SCHELK_DIR=$DATADIR SCHELK_STATE=$SF_WORK/.schelk/state.json shadowfork/inplace-schelk mount + { echo 'config<> "$GITHUB_OUTPUT" + + - name: Pre-run (fill) + uses: ethpandaops/benchmarkoor@master + env: + BENCHMARKOOR_BUILDER_RUN_TIMEOUT: 2400m + with: + mode: build + github-token: ${{ secrets.GITHUB_TOKEN }} + image: ${{ inputs.benchmarkoor-image }} + upload-artifacts: "false" + upload-build-artifacts: "false" + tmp-dir: ${{ env.SF_WORK }}/tmp + build-config: ${{ steps.cfg.outputs.config }} + build-args: --force + + - name: Ramp-down, journal flush, compaction, digest + env: + GAS_TARGET: ${{ inputs.gas-target }} + run: | + source shadowfork/lib.sh; source "$SF_WORK/anchor.env" + sudo touch "$DATADIR/.advanced" + sudo umount "$DATADIR" + shadowfork/finalize.sh + + - name: Publish + if: inputs.publish + run: shadowfork/publish.sh + + - name: Verify uploads + if: inputs.publish + run: | + source shadowfork/lib.sh + END=$(jq .end_block_number "$SF_WORK/bundle/pre_run_bundle/pre-run.meta.json") + shadowfork/verify.sh "mainnet/geth/$END/snapshot.tar.zst" "$SF_WORK/publish-geth.blocks" 3 + shadowfork/verify.sh "bundles/mainnet-$END/bundle.tar.zst" "$SF_WORK/publish-bundle.blocks" 1 + + - name: Summary + if: always() + run: | + m=$SF_WORK/bundle/pre_run_bundle/pre-run.meta.json + [ -f "$m" ] || exit 0 + { + echo "### shadowfork image (geth)" + echo '```json'; cat "$m"; echo '```' + [ -f "$SF_WORK/publish-geth.blocks" ] && echo "geth image: $(awk '$2=="total"{print $1}' "$SF_WORK/publish-geth.blocks") bytes" + echo + echo "Next: replay the bundle on the other clients, then metadata and \`latest\` (shadowfork/README.md)." + } >> "$GITHUB_STEP_SUMMARY" + + # The runner is shared (syncoor): never leave 1.5 TB behind. + - name: Clean up + if: always() + run: | + docker rm -f sf-geth >/dev/null 2>&1 || true + [ -z "${SF_WORK:-}" ] || { sudo umount "$SF_WORK/geth-datadir" 2>/dev/null || true; sudo rm -rf "$SF_WORK"; } diff --git a/shadowfork/.gitignore b/shadowfork/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/shadowfork/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/shadowfork/README.md b/shadowfork/README.md new file mode 100644 index 0000000..ba0c3da --- /dev/null +++ b/shadowfork/README.md @@ -0,0 +1,71 @@ +# Shadowfork benchmark images + +Post-Gloas mainnet datadirs for every EL, ready for a mainnet shadowfork (msf-2) to boot and +benchmark without a prefill of its own. Published to `ethpandaops-shadowfork-images`, read at +`https://shadowfork-images.ethpandaops.io/mainnet///snapshot.tar.zst` with the +usual `_snapshot_eth_getBlockByNumber.json` / `_snapshot_metadata.json` / `latest` layout, so +ethereum-package's `network_sync_base_url` works unchanged. + +An image is a fresh `snapshots.ethpandaops.io` mainnet snapshot advanced by one pre-run that: + +1. funds the 25,000-account pool and buildoor's wallet, and deploys the EIP-8282 contracts in two + **Osaka** blocks (a strict client rejects every Amsterdam block without them); +2. crosses **Amsterdam** 3 s after the snapshot head, then ramps the gas limit to 2 G; +3. deploys `test_account_access`'s receivers at **24 KiB and 64 KiB**: SAME_MAX, DIFF_MAX and + JUMPDEST at both sizes, MINIMAL once, plus the 7702 delegations, in one test + (`test_deploy_existing_contracts_all_sizes`). 100k per variant covers benchmarks up to ~300 Mgas, + 1 Ggas needs ~321k; +4. walks the gas limit back down to the target (200 M) with empty blocks. + +geth fills; every other client **replays** the recorded bundle, so all images are one chain with one +head hash, and a client is only published if its digest (hash, stateRoot, gasUsed, BAL hash at ~200 +heights) is identical to geth's. + +## A clean image: nothing recent may be faster to read than old mainnet state + +Benchmarks against a prefilled snapshot found the contracts deployed after the snapshot in higher +DB tiers, and for some clients still warm in the journal, so they read faster than real mainnet +state would. Each client is cleaned before it is packed: + +| client | journal / hot layer | compaction | +|---|---|---| +| geth | path scheme keeps 128 diff layers plus an unflushed write buffer (up to 256 MiB) and writes both to `triedb/merkle.journal` on a clean stop, which the next start reads back into memory. The 2026-10-08 image journaled **1,354** blocks: the last ~1,200 fill blocks never reached pebble. The ramp-down runs with `--cache.gc=0` (a 0-byte buffer), so every block that leaves the diff layers is flushed, and `finalize.sh` fails unless the journal holds exactly the last 128 (empty) blocks | `geth db compact` | +| erigon | `seg retire` | `erigon db compact` | +| nethermind, besu | none | RocksDB `ldb compact` per column family, offline (neither ships a compaction command) | +| reth | none: MDBX, persisted every block | none needed | + +The geth mechanism was verified on a dev chain (v1.17.7): 176 blocks of state journaled as +`layers=176`; restarted with `--cache.gc=0` (`buffer=0.00B`), ~150 later blocks left `layers=128`. + +## Running it + +`shadowfork-image-build` (workflow_dispatch) does the geth half on a `synctest` `Disk4TB` runner: +snapshot, pre-run, ramp-down + flush + compaction, reference digest, then uploads +`mainnet/geth//` and `bundles/mainnet-/bundle.tar.zst` (payloads, genesis, anchor, +`digest-geth.txt`, head block). It never writes `latest`. + +The other clients are replayed separately, one machine each, from that bundle: download the +client's `snapshots.ethpandaops.io/mainnet//` snapshot, replay every payload +(each must be VALID), check that it reopens at the bundle's head, clean it as above, compare its +digest against `digest-geth.txt`, then pack and upload. Once all five are up and verified, write +each client's `_snapshot_metadata.json` and head block, and `latest` last. + +Requirements learned the hard way: +- Open each datadir with the **release** that wrote the snapshot and that the devnets run, not a + devnet build: the glamsterdam-devnet-8 images trailed by a major version (nethermind 1.40 vs 2.1). +- The fill must not hold its fixture in memory: each 64 KiB deploy's code is also in its block + access list, so the bundle is ~55 GB and an in-memory fixture passed 200 GiB. The default EEST + ref carries skylenet's spill-to-disk fixes. +- Every long step is its own process: no host may auto-upgrade or restart services mid-build + (unattended-upgrades restarted a unit and killed a 1 TB upload with it). + +## Files + +| file | does | +|---|---| +| `prepare.sh` | download the geth snapshot (parallel ranged GETs), genesis with Amsterdam at head + 3 s | +| `prerun-config.sh` | the `benchmarkoor build` pre-run config (predeploy block lifted from the devnet-8 prefill) | +| `inplace-schelk` | schelk's CLI over a bind-mounted directory, so the pre-run advances the datadir in place | +| `rampdown.py` | empty blocks down to the gas target, appended to the bundle | +| `finalize.sh` | ramp-down with a 0-byte buffer, journal check, compaction, reopen, reference digest | +| `publish.sh`, `blockhash.py`, `verify.sh` | stream to the bucket hashing every GiB; re-read random GiBs | diff --git a/shadowfork/blockhash.py b/shadowfork/blockhash.py new file mode 100755 index 0000000..599f8f8 --- /dev/null +++ b/shadowfork/blockhash.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""sha256 of every 1 GiB block of stdin: ' ' per block, then ' total'.""" +import hashlib, sys + +BLOCK = 1 << 30 +i = total = 0 +h, n = hashlib.sha256(), 0 +for chunk in iter(lambda: sys.stdin.buffer.read(1 << 22), b""): + total += len(chunk) + while chunk: + take = chunk[: BLOCK - n] + h.update(take); n += len(take); chunk = chunk[len(take):] + if n == BLOCK: + print(i, h.hexdigest(), flush=True); i += 1; h, n = hashlib.sha256(), 0 +if n: + print(i, h.hexdigest(), flush=True) +print(total, "total", flush=True) diff --git a/shadowfork/finalize.sh b/shadowfork/finalize.sh new file mode 100755 index 0000000..e63d679 --- /dev/null +++ b/shadowfork/finalize.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# After the pre-run: walk the gas limit down, push the fill out of geth's journal, compact, and +# write the reference digest the replaying clients are checked against. +# GAS_TARGET=200000000 shadowfork/finalize.sh +# +# Why each step (shadowfork/README.md, "A clean image"): +# - geth's path scheme keeps the last 128 blocks as in-memory diff layers and everything older but +# unflushed in a write buffer (up to 256 MiB); a clean stop writes both to triedb/merkle.journal, +# and the next start reads them back into memory. The pre-run's last ~1,200 blocks of deployed +# contracts sat there (2026-10-08), so a benchmark found them in RAM rather than on disk. +# --cache.gc=0 makes the buffer 0 bytes: every block that leaves the diff layers is flushed. +# - Freshly written keys sit in the top LSM levels while mainnet's state is in the bottom ones; +# `geth db compact` rewrites everything to the bottom level. +source "$(dirname "$0")/lib.sh" +source "$WORK/anchor.env" +GAS_TARGET=${GAS_TARGET:-200000000} +B=$WORK/bundle/pre_run_bundle +[ -f "$B/pre-run.meta.json" ] || die "no bundle at $B" +lines=$(wc -l < "$B/pre-run.request"); payloads=$(jq .payloads "$B/pre-run.meta.json") +[ "$lines" -eq $((payloads * 2)) ] || die "bundle has $lines lines for $payloads payloads" + +log "=== ramp-down to $GAS_TARGET with a 0-byte write buffer ===" +boot_geth --cache.gc=0 --miner.gaslimit="$GAS_TARGET" +docker logs sf-geth 2>&1 | grep -q 'buffer=0.00B' || die "geth did not start with a 0-byte write buffer" +python3 "$SF_DIR/rampdown.py" "http://127.0.0.1:$P_RPC" "http://127.0.0.1:$P_ENGINE" "$WORK/jwtsecret" "$WORK/bundle" "$GAS_TARGET" +stop_geth rampdown +# The journal may hold the 128 diff layers of the last (empty) blocks and nothing else. +grep -E 'Persisting dirty state' "$WORK/geth-rampdown.log" | tail -1 +grep -E 'Persisting dirty state.*layers=128$' "$WORK/geth-rampdown.log" >/dev/null \ + || die "geth journaled more than the 128 diff layers: the fill is still in memory" + +log "=== geth db compact ===" +docker run --rm -v "$DATADIR:/data" "$GETH_IMAGE" db compact --datadir=/data 2>&1 | tail -5 + +log "=== reopen and reference digest ===" +END=$(jq .end_block_number "$B/pre-run.meta.json"); END_HASH=$(jq -r .end_block_hash "$B/pre-run.meta.json") +boot_geth +h=$(rpc eth_getBlockByNumber '["latest",false]') +[ "$(( $(jq -r .result.number <<<"$h") ))" = "$END" ] && [ "$(jq -r .result.hash <<<"$h")" = "$END_HASH" ] \ + || die "geth reopens at $(jq -c '.result | [.number, .hash]' <<<"$h"), not $END $END_HASH" +# The same heights lib/replay.py digests on a replaying client: first payload, every n/200th. +start=$(head -1 "$B/pre-run.request" | jq -r '.params[0].blockNumber'); start=$((start)) +n=$(jq .payloads "$B/pre-run.meta.json"); step=$(( n / 200 > 0 ? n / 200 : 1 )) +for ((b = start; b < start + n; b += step)); do + r=$(rpc eth_getBlockByNumber "[\"$(printf 0x%x $b)\",false]") + jq -r --argjson b "$b" '"\($b) \(.result.hash) \(.result.stateRoot) \(.result.gasUsed) \(.result.blockAccessListHash // "none")"' <<<"$r" +done > "$WORK/digest-geth.txt" +rpc eth_getBlockByNumber "[\"$(printf 0x%x "$END")\",true]" > "$WORK/_snapshot_eth_getBlockByNumber.json" +stop_geth final +log "image head $END $END_HASH, gas limit $(( $(jq -r .result.gasLimit "$WORK/_snapshot_eth_getBlockByNumber.json") )), $(wc -l < "$WORK/digest-geth.txt") digest heights" diff --git a/shadowfork/inplace-schelk b/shadowfork/inplace-schelk new file mode 100755 index 0000000..f2a0d78 --- /dev/null +++ b/shadowfork/inplace-schelk @@ -0,0 +1,29 @@ +#!/bin/bash +# schelk's CLI, as benchmarkoor drives it (pkg/datadir/schelk.go), on a plain directory: the +# pre-run advances it in place and nothing can roll it back. For a disposable builder whose disk +# cannot hold a copy (build/README.md); tools/zfs-schelk keeps a pristine @baseline instead. +# +# SCHELK_DIR the datadir (required); bind-mounted onto itself so it is in /proc/mounts +# SCHELK_STATE state file benchmarkoor reads (default /var/lib/schelk/state.json) +# +# restore/recover before a pre-run are no-ops on a pristine dir. Once 01-prerun.sh has marked it +# .advanced they fail: a rollback that cannot happen must not report success. +set -euo pipefail +DIR=${SCHELK_DIR:?SCHELK_DIR is not set} +STATE=${SCHELK_STATE:-/var/lib/schelk/state.json} +die() { echo "inplace-schelk: $*" >&2; exit 1; } + +mkdir -p "$(dirname "$STATE")" +exec 9>"${STATE%.json}.lock" +# benchmarkoor retries on exactly this text while another schelk holds the lock. +flock -n 9 || die "Another schelk process is already running" +[ -d "$DIR" ] || die "no directory $DIR" +mountpoint -q "$DIR" || sudo mount --bind "$DIR" "$DIR" + +case "${1:-}" in + init|mount|promote) ;; + restore|recover|full-recover) + [ ! -e "$DIR/.advanced" ] || die "$DIR was advanced in place and has no baseline to $1 to" ;; + *) die "unsupported verb '${1:-}' (init|restore|recover|full-recover|mount|promote)" ;; +esac +printf '{"mount_point":"%s","is_mounted":true,"backend":"inplace"}\n' "$DIR" > "$STATE" diff --git a/shadowfork/lib.sh b/shadowfork/lib.sh new file mode 100755 index 0000000..ff69d8a --- /dev/null +++ b/shadowfork/lib.sh @@ -0,0 +1,39 @@ +# shellcheck shell=bash disable=SC2034 # sourced: the settings are used by the scripts that source it +# Shared settings for the shadowfork image build (see shadowfork/README.md). +set -euo pipefail +SF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORK=${SF_WORK:?SF_WORK must point at a directory on the big disk} +SNAP=${SF_SNAPSHOTS:-https://snapshots.ethpandaops.io} +BUCKET=${SF_BUCKET:-ethpandaops-shadowfork-images} +PUBLIC=${SF_PUBLIC:-https://shadowfork-images.ethpandaops.io} +# The image is opened by the release that wrote the snapshot and that the devnets run. +GETH_IMAGE=${GETH_IMAGE:-ethereum/client-go:v1.17.7} +P_RPC=19545 P_ENGINE=19551 +mkdir -p "$WORK" + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } +rpc() { # rpc [params-json] + curl -sf --max-time 60 -X POST -H 'content-type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$1\",\"params\":${2:-[]}}" "http://127.0.0.1:$P_RPC" +} +r2() { # rclone against the images bucket with the repo's S3 credentials + RCLONE_CONFIG_R2_TYPE=s3 RCLONE_CONFIG_R2_PROVIDER=Cloudflare RCLONE_CONFIG_R2_ENDPOINT=${S3_ENDPOINT_URL:?} \ + RCLONE_CONFIG_R2_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:?} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:?} \ + rclone "$@" +} +boot_geth() { # boot_geth : the image datadir on the fixed ports, no peers + source "$WORK/anchor.env" + docker rm -f sf-geth >/dev/null 2>&1 || true + docker run -d --name sf-geth --network host -v "$DATADIR:/data" -v "$GENESIS:/genesis.json:ro" \ + -v "$WORK/jwtsecret:/jwtsecret:ro" "$GETH_IMAGE" \ + --datadir=/data --override.genesis=/genesis.json --override.amsterdam="$AMS_TS" \ + --http --http.addr=127.0.0.1 --http.port=$P_RPC --http.api=eth,debug,testing \ + --authrpc.addr=127.0.0.1 --authrpc.port=$P_ENGINE --authrpc.jwtsecret=/jwtsecret \ + --syncmode=full --maxpeers=0 --nodiscover "$@" >/dev/null + for _ in $(seq 120); do rpc eth_blockNumber >/dev/null 2>&1 && return 0; sleep 5; done + docker logs --tail 20 sf-geth; die "geth did not open its RPC" +} +stop_geth() { # graceful: a killed geth never writes its trie journal and reopens short of its head + docker stop -t 1800 sf-geth >/dev/null; docker logs sf-geth > "$WORK/geth-$1.log" 2>&1; docker rm sf-geth >/dev/null +} diff --git a/shadowfork/pfetch.sh b/shadowfork/pfetch.sh new file mode 100755 index 0000000..8c00217 --- /dev/null +++ b/shadowfork/pfetch.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# pfetch.sh [workers] [chunk-bytes] : write to stdout, fetched as parallel +# ranged GETs and emitted in order. One stream from R2 via Cloudflare tops out ~95 MB/s; +# 16 ranges measured 239-349 MB/s. POSIX sh + curl, so it runs in ethereum-package's +# alpine downloader: `pfetch.sh "$URL" | tar -I zstd -xf - -C "$DIR"`. +# At most 2 x workers chunks are staged in $TMPDIR; a chunk is retried until complete. +set -eu +URL=$1; P=${2:-16}; C=${3:-67108864} +TOTAL=$(curl -sfIL "$URL" | tr -d '\r' | awk 'tolower($1)=="content-length:"{n=$2} END{print n}') +[ -n "$TOTAL" ] || { echo "pfetch: no Content-Length for $URL" >&2; exit 1; } +[ -z "${PFETCH_BYTES:-}" ] || [ "$PFETCH_BYTES" -ge "$TOTAL" ] || TOTAL=$PFETCH_BYTES # testing: a prefix only +N=$(( (TOTAL + C - 1) / C )) +T=$(mktemp -d); echo 0 > "$T/emitted" +PIDS="" # our workers only: `kill 0` would also hit the tar we are piped into +trap 'kill $PIDS 2>/dev/null || true; rm -rf "$T"' EXIT INT TERM + +fetch() { # fetch : until the chunk file holds exactly its range + s=$(( $1 * C )); e=$(( s + C - 1 )); [ "$e" -lt "$TOTAL" ] || e=$(( TOTAL - 1 )) + want=$(( e - s + 1 )); tries=0 + while :; do + # --max-filesize: a cold, cache-eligible object can answer a Range with 200 and the + # whole body (measured on snapshots.ethpandaops.io); refuse it before a byte lands. + curl -sfL --connect-timeout 20 --speed-limit 1024 --speed-time 60 --max-filesize "$want" \ + -r "$s-$e" -o "$T/$1.part" "$URL" || true + [ "$(wc -c 2>/dev/null < "$T/$1.part" || echo 0)" -eq "$want" ] && { mv "$T/$1.part" "$T/$1"; return; } + tries=$(( tries + 1 )); [ "$tries" -lt 50 ] || { echo "pfetch: chunk $1 failed $tries times" >&2; touch "$T/failed"; return 1; } + sleep 2 + done +} + +k=0 +while [ "$k" -lt "$P" ]; do + { ( i=$k + while [ "$i" -lt "$N" ]; do + while [ $(( i - $(cat "$T/emitted") )) -ge $(( 2 * P )) ]; do sleep 0.1; done + fetch "$i" || exit 1 + i=$(( i + P )) + done ) || touch "$T/failed"; } & # a worker that dies must fail the stream, not hang it + PIDS="$PIDS $!"; k=$(( k + 1 )) +done + +i=0 +while [ "$i" -lt "$N" ]; do + while [ ! -f "$T/$i" ]; do [ ! -f "$T/failed" ] || exit 1; sleep 0.05; done + cat "$T/$i"; rm -f "$T/$i"; i=$(( i + 1 )) + echo "$i" > "$T/emitted.tmp"; mv "$T/emitted.tmp" "$T/emitted" # atomic: a torn read kills a worker +done +wait diff --git a/shadowfork/prepare.sh b/shadowfork/prepare.sh new file mode 100755 index 0000000..388db17 --- /dev/null +++ b/shadowfork/prepare.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Download the anchor mainnet geth snapshot and write the pre-run's genesis. +# shadowfork/prepare.sh [block|latest] +# Amsterdam is scheduled 3 s after the snapshot head: the pre-run's two funding/predeploy blocks +# are Osaka (the EIP-8282 contracts must exist before the fork), everything after is Amsterdam. +source "$(dirname "$0")/lib.sh" +BLOCK=${1:-latest} +[ "$BLOCK" != latest ] || BLOCK=$(curl -sf "$SNAP/mainnet/geth/latest" | tr -d '[:space:]') +[[ "$BLOCK" =~ ^[0-9]+$ ]] || die "no anchor block ($BLOCK)" +DATADIR=$WORK/geth-datadir +URL=$SNAP/mainnet/geth/$BLOCK/snapshot.tar.zst + +# The pre-run advances the datadir in place, so the disk needs the extracted snapshot plus the +# fill's delta, its fixtures and the bundle (55 GB at 100k receivers per variant). +need=$(( $(curl -sf "$SNAP/mainnet/geth/$BLOCK/_snapshot_metadata.json" | jq -r .data_size_bytes) + 400 * 1024**3 )) +free=$(( $(df -B1 --output=avail "$WORK" | tail -1) )) +[ "$free" -gt "$need" ] || die "$WORK has $((free / 1024**3)) GiB free, needs $((need / 1024**3))" + +if [ ! -f "$DATADIR/.complete" ]; then + sudo rm -rf "$DATADIR"; mkdir -p "$DATADIR/geth" + log "downloading mainnet/geth/$BLOCK ($(curl -sfI "$URL" | tr -d '\r' | awk 'tolower($1)=="content-length:"{printf "%.0f GiB", $2/2^30}'))" + # The published snapshot has no geth/ prefix; geth wants /geth/chaindata. + "$SF_DIR/pfetch.sh" "$URL" "${PFETCH_WORKERS:-16}" | zstd -d --stdout | sudo tar -xf - -C "$DATADIR/geth" \ + || die "download/extract failed" + sudo touch "$DATADIR/.complete" +fi + +HEAD=$(curl -sf "$SNAP/mainnet/geth/$BLOCK/_snapshot_eth_getBlockByNumber.json") +HEAD_TS=$(( $(jq -r '.result.timestamp' <<<"$HEAD") )) +AMS_TS=$((HEAD_TS + 3)) +GENESIS=$WORK/genesis.json +# Only forks not yet active at the head may move: moving an active one makes geth rewind. +curl -sf https://raw.githubusercontent.com/eth-clients/mainnet/refs/heads/main/metadata/genesis.json \ + | jq --argjson h "$HEAD_TS" --argjson a "$AMS_TS" ' + .config |= with_entries(if (.key|test("^(osaka|bpo[0-9]+)Time$")) and (.value > $h) then .value = $h + 1 else . end) + | .config.amsterdamTime = $a' > "$GENESIS" +[ "$(jq -r '[.config | to_entries[] | select(.key|test("Time$")) | .value] | max' "$GENESIS")" = "$AMS_TS" ] \ + || die "a parent fork is scheduled after amsterdam; this anchor cannot cross it" +[ -f "$WORK/jwtsecret" ] || openssl rand -hex 32 > "$WORK/jwtsecret" +cat > "$WORK/anchor.env" <.blocks (sha256 of every GiB of the uploaded stream) for verify.sh. +# Never writes `latest`: that waits until every client's image is up (shadowfork/README.md). +source "$(dirname "$0")/lib.sh" +source "$WORK/anchor.env" +B=$WORK/bundle/pre_run_bundle +END=$(jq .end_block_number "$B/pre-run.meta.json") +up() { # up : stdin -> zstd -> hashed -> bucket + zstd -6 -T0 -q | tee >(python3 "$SF_DIR/blockhash.py" > "$WORK/publish-$1.blocks") \ + | r2 rcat --s3-chunk-size 256M --s3-upload-concurrency 8 "r2:$BUCKET/$2" + sleep 5 # the process substitution may still be flushing + grep -q total "$WORK/publish-$1.blocks" || die "$1: upload stream incomplete" +} +docker ps --format '{{.Names}}' | grep -qx sf-geth && die "geth is still running" + +log "=== geth image -> mainnet/geth/$END ===" +# The geth/ prefix is load-bearing: /geth/triedb holds the state geth reopens with. +sudo tar --sort=name -C "$DATADIR" --exclude=geth/nodekey --exclude=geth/LOCK --exclude=geth/nodes \ + --exclude='geth/_snapshot_*' -cf - geth | up geth "mainnet/geth/$END/snapshot.tar.zst" +r2 copyto "$WORK/_snapshot_eth_getBlockByNumber.json" "r2:$BUCKET/mainnet/geth/$END/_snapshot_eth_getBlockByNumber.json" + +log "=== bundle -> bundles/mainnet-$END ===" +# Everything a replaying client needs: the payloads, the genesis they were built against, the +# anchor, and geth's digest to compare its own against. +tar --sort=name -C "$WORK" -cf - anchor.env genesis.json digest-geth.txt _snapshot_eth_getBlockByNumber.json \ + bundle/pre_run_bundle/pre-run.request bundle/pre_run_bundle/pre-run.meta.json | up bundle "bundles/mainnet-$END/bundle.tar.zst" +log "published head $END: geth image $(awk '$2=="total"{print $1}' "$WORK/publish-geth.blocks") B, bundle $(awk '$2=="total"{print $1}' "$WORK/publish-bundle.blocks") B" diff --git a/shadowfork/rampdown.py b/shadowfork/rampdown.py new file mode 100755 index 0000000..db0f917 --- /dev/null +++ b/shadowfork/rampdown.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Append empty blocks to the image until its gas limit is the target, and record them in the bundle. + +The pre-run leaves the head at its 2 G bump target; a devnet that wants 200 M would otherwise +spend ~2,400 blocks walking down 1/1024 per block. The blocks are built the way benchmarkoor +builds its own (testing_buildBlockV1, then engine_newPayloadV5 + engine_forkchoiceUpdatedV3), +with geth started at --miner.gaslimit=, and appended to pre-run.request so every replayed +client reaches the same head. At least MIN_BLOCKS are built even once the target is reached: they +push the fill out of geth's 128 in-memory diff layers. + +Usage: rampdown.py +""" +import base64, hashlib, hmac, json, sys, time, urllib.request + +rpc_url, engine_url, jwt_path, bdir, target = sys.argv[1:6] +target = int(target) +MIN_BLOCKS = 256 +EXTRA_DATA = "0x62656e63686d61726b6f6f72" # "benchmarkoor", as on its own blocks +secret = bytes.fromhex(open(jwt_path).read().strip().removeprefix("0x")) + +def jwt(): + b = lambda x: base64.urlsafe_b64encode(x).rstrip(b"=") + h, p = b(b'{"alg":"HS256","typ":"JWT"}'), b(json.dumps({"iat": int(time.time())}).encode()) + return (h + b"." + p + b"." + b(hmac.new(secret, h + b"." + p, hashlib.sha256).digest())).decode() + +def call(url, method, params, auth=False): + hdr = {"Content-Type": "application/json"} + if auth: + hdr["Authorization"] = "Bearer " + jwt() + body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode() + r = json.load(urllib.request.urlopen(urllib.request.Request(url, data=body, headers=hdr), timeout=120)) + if "error" in r: + sys.exit(f"{method}: {r['error']}") + return r["result"] + +req_path, meta_path = f"{bdir}/pre_run_bundle/pre-run.request", f"{bdir}/pre_run_bundle/pre-run.meta.json" +meta = json.load(open(meta_path)) +head = call(rpc_url, "eth_getBlockByNumber", ["latest", False]) +if int(head["number"], 16) != meta["end_block_number"] or head["hash"] != meta["end_block_hash"]: + sys.exit(f"geth head {int(head['number'], 16)} is not the bundle end {meta['end_block_number']}") + +i, built, prev_gl = meta["payloads"], 0, int(head["gasLimit"], 16) +with open(req_path, "a") as f: + while built < MIN_BLOCKS or int(head["gasLimit"], 16) > target: + attrs = {"timestamp": hex(int(head["timestamp"], 16) + 1), "prevRandao": head["hash"], + "suggestedFeeRecipient": "0x" + "00" * 20, "withdrawals": [], + "parentBeaconBlockRoot": head["hash"]} + # The image is past Amsterdam (V5, slot numbers); an Osaka chain takes V4 (used by the test). + amsterdam = head.get("slotNumber") is not None + if amsterdam: + attrs["slotNumber"] = hex(int(head["slotNumber"], 16) + 1) + method = "engine_newPayloadV5" if amsterdam else "engine_newPayloadV4" + res = call(rpc_url, "testing_buildBlockV1", [head["hash"], attrs, [], EXTRA_DATA]) + payload = res.get("executionPayload", res) + params = [payload, [], head["hash"], res.get("executionRequests") or []] + st = call(engine_url, method, params, auth=True) + if st["status"] != "VALID": + sys.exit(f"block {int(payload['blockNumber'], 16)}: newPayload {st}") + fc = {"finalizedBlockHash": payload["blockHash"], "headBlockHash": payload["blockHash"], + "safeBlockHash": payload["blockHash"]} + call(engine_url, "engine_forkchoiceUpdatedV3", [fc, None], auth=True) + i += 1 + f.write(json.dumps({"id": i, "jsonrpc": "2.0", "method": method, "params": params}, + separators=(",", ":")) + "\n") + f.write(json.dumps({"id": i, "jsonrpc": "2.0", "method": "engine_forkchoiceUpdatedV3", "params": [fc, None]}, + separators=(",", ":")) + "\n") + head = call(rpc_url, "eth_getBlockByNumber", ["latest", False]) + gl = int(head["gasLimit"], 16) + if built == 0 and gl >= prev_gl and gl > target: + sys.exit(f"gas limit did not fall ({prev_gl} -> {gl}): is geth running with --miner.gaslimit={target}?") + built += 1 + if built % 500 == 0: + print(f" {built} blocks, gas limit {gl}", flush=True) + +meta.update(payloads=i, end_block_number=int(head["number"], 16), end_block_hash=head["hash"], + ramp_down_blocks=built, end_gas_limit=int(head["gasLimit"], 16)) +json.dump(meta, open(meta_path, "w"), indent=2) +print(f"ramp-down: {built} blocks to gas limit {int(head['gasLimit'], 16)}, head {meta['end_block_number']} {head['hash']}") diff --git a/shadowfork/verify.sh b/shadowfork/verify.sh new file mode 100755 index 0000000..2735d93 --- /dev/null +++ b/shadowfork/verify.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Check an uploaded object against the block hashes publish.sh took of the stream it sent: its size, +# then N random 1 GiB blocks. Images are read through the public host as consumers read them (each +# Range must be a 206); bundles/ is not public (WAF), so it is read with the bucket credentials. +# shadowfork/verify.sh [N=3] +source "$(dirname "$0")/lib.sh" +obj=${1:?object} f=${2:?blocks file} n=${3:-3} +want=$(awk '$2=="total"{print $1}' "$f") +if [[ $obj == mainnet/* ]]; then + got=$(curl -sfI "$PUBLIC/$obj" | tr -d '\r' | awk 'tolower($1)=="content-length:"{print $2}') +else + got=$(r2 size --json "r2:$BUCKET/$obj" | jq .bytes) +fi +[ "$got" = "$want" ] || die "$obj: size $got, uploaded $want" +blocks=$(grep -vc total "$f") +for i in $(shuf -i 0-$((blocks - 1)) -n "$(( n < blocks ? n : blocks ))"); do + s=$((i << 30)); len=$(( (1 << 30) )); [ $((s + len)) -le "$want" ] || len=$((want - s)) + if [[ $obj == mainnet/* ]]; then + code=$(curl -s -o "$WORK/vb.$i" -w '%{http_code}' -r "$s-$((s + len - 1))" "$PUBLIC/$obj") + [ "$code" = 206 ] || die "$obj block $i: HTTP $code" + else + r2 cat --offset "$s" --count "$len" "r2:$BUCKET/$obj" > "$WORK/vb.$i" + fi + [ "$(sha256sum < "$WORK/vb.$i" | cut -d' ' -f1)" = "$(awk -v i="$i" '$1==i{print $2}' "$f")" ] \ + || die "$obj block $i: hash mismatch" + rm -f "$WORK/vb.$i"; log "$obj block $i OK" +done +log "$obj: $want bytes, size and random blocks match" From 1e383395e42f32b371281ca38cf0596158752eae Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 14:02:01 +0200 Subject: [PATCH 02/14] shadowfork-image-build: EEST from upstream forks/amsterdam + #3648 + #3304 qu0b/bench-images-v3 replaces the skylenet-based v2: upstream forks/amsterdam (2026-09-25) with the two open upstream PRs the fill needs, ethereum/execution-specs#3648 (spill payload lists to disk) and #3304 (the deployment script), plus the combined-sizes test and JUMPDEST at 24 KiB. --- .github/workflows/shadowfork-image-build.yaml | 4 ++-- shadowfork/README.md | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/shadowfork-image-build.yaml b/.github/workflows/shadowfork-image-build.yaml index c752bc0..f5579c5 100644 --- a/.github/workflows/shadowfork-image-build.yaml +++ b/.github/workflows/shadowfork-image-build.yaml @@ -23,8 +23,8 @@ on: description: "execution-specs repo for the fill" default: "https://github.com/qu0b/execution-specs.git" eest-ref: - description: "execution-specs ref: needs the combined-sizes setup test and skylenet's fixture spill" - default: "qu0b/bench-images-v2" + description: "execution-specs ref: upstream forks/amsterdam + #3648 (fixture spill) + #3304 (deploy script) + the combined-sizes test" + default: "qu0b/bench-images-v3" geth-image: description: "geth release that fills and opens the image (the one the devnets run)" default: "ethereum/client-go:v1.17.7" diff --git a/shadowfork/README.md b/shadowfork/README.md index ba0c3da..c5aa977 100644 --- a/shadowfork/README.md +++ b/shadowfork/README.md @@ -55,7 +55,9 @@ Requirements learned the hard way: devnet build: the glamsterdam-devnet-8 images trailed by a major version (nethermind 1.40 vs 2.1). - The fill must not hold its fixture in memory: each 64 KiB deploy's code is also in its block access list, so the bundle is ~55 GB and an in-memory fixture passed 200 GiB. The default EEST - ref carries skylenet's spill-to-disk fixes. + ref (`qu0b/execution-specs@qu0b/bench-images-v3`) is upstream `forks/amsterdam` plus the two open + upstream PRs this needs, ethereum/execution-specs#3648 (spill payload lists to disk) and #3304 + (the deployment script), plus the combined-sizes test and JUMPDEST at 24 KiB. - Every long step is its own process: no host may auto-upgrade or restart services mid-build (unattended-upgrades restarted a unit and killed a 1 TB upload with it). From 5705676cf34c3807cc074c9d613bf288fb5cd28e Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 16:50:12 +0200 Subject: [PATCH 03/14] benchmarkoor-snapshot: a jochemnet release as a shadowfork snapshot Replaces shadowfork-image-build and its scripts. geth replays the release's pre-run (base_bundle) and walks the gas limit down (fill: false), every other client replays release + ramp, each datadir is compacted (db compact) and exported (snapshot export) to /jochemnet///, and latest is written once all are up. Runs on the jochemnet hosts and recovers their schelk scratch afterwards. --- .github/scripts/release-pre-run-bundle.sh | 24 ++ .github/scripts/snapshot-export.sh | 23 ++ .github/workflows/benchmarkoor-snapshot.yaml | 322 ++++++++++++++++++ .github/workflows/shadowfork-image-build.yaml | 145 -------- README.md | 23 ++ configs/snapshots/jochemnet/v1/builder.yaml | 100 ++++++ shadowfork/.gitignore | 1 - shadowfork/README.md | 73 ---- shadowfork/blockhash.py | 17 - shadowfork/finalize.sh | 50 --- shadowfork/inplace-schelk | 29 -- shadowfork/lib.sh | 39 --- shadowfork/pfetch.sh | 48 --- shadowfork/prepare.sh | 47 --- shadowfork/prerun-config.sh | 58 ---- shadowfork/publish.sh | 29 -- shadowfork/rampdown.py | 78 ----- shadowfork/verify.sh | 28 -- 18 files changed, 492 insertions(+), 642 deletions(-) create mode 100755 .github/scripts/release-pre-run-bundle.sh create mode 100755 .github/scripts/snapshot-export.sh create mode 100644 .github/workflows/benchmarkoor-snapshot.yaml delete mode 100644 .github/workflows/shadowfork-image-build.yaml create mode 100644 configs/snapshots/jochemnet/v1/builder.yaml delete mode 100644 shadowfork/.gitignore delete mode 100644 shadowfork/README.md delete mode 100755 shadowfork/blockhash.py delete mode 100755 shadowfork/finalize.sh delete mode 100755 shadowfork/inplace-schelk delete mode 100755 shadowfork/lib.sh delete mode 100755 shadowfork/pfetch.sh delete mode 100755 shadowfork/prepare.sh delete mode 100755 shadowfork/prerun-config.sh delete mode 100755 shadowfork/publish.sh delete mode 100755 shadowfork/rampdown.py delete mode 100755 shadowfork/verify.sh diff --git a/.github/scripts/release-pre-run-bundle.sh b/.github/scripts/release-pre-run-bundle.sh new file mode 100755 index 0000000..875639a --- /dev/null +++ b/.github/scripts/release-pre-run-bundle.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Fetch a benchmarkoor-release's pre-run bundle into /pre_run_bundle, checked +# against the release's sha256. Prints the bundle's pre-run.meta.json. +# release-pre-run-bundle.sh +set -euo pipefail +tag=${1:?release tag} dest=${2:?dest dir} +base="https://github.com/${GITHUB_REPOSITORY:-ethpandaops/benchmarkoor-tests}/releases/download/$tag" +dl="$dest/download" +mkdir -p "$dl" + +curl -fsSL --retry 5 "$base/manifest.json" -o "$dl/manifest.json" +# Every client's run replays the geth pre-run (test-source.*.runner.yaml), so it is the only one. +asset=$(jq -ce '.assets[] | select(.client == "geth")' "$dl/manifest.json") +tarball=$(jq -r .tarball <<<"$asset") +mapfile -t files < <(jq -r '.files[]' <<<"$asset") + +for f in "${files[@]}"; do curl -fsSL --retry 5 "$base/$f" -o "$dl/$f"; done +# Assets over 2 GB are split into .part-NNN; an unsplit asset is the tarball itself. +[ "${#files[@]}" -eq 1 ] && [ "${files[0]}" = "$tarball" ] || { (cd "$dl" && cat "${files[@]}") > "$dl/$tarball"; } +echo "$(jq -r .sha256 <<<"$asset") $dl/$tarball" | sha256sum -c --quiet - + +tar -xzf "$dl/$tarball" -C "$dest" --strip-components=3 benchmarkoor-build-artifacts/pre-runs/geth/pre_run_bundle +rm -rf "$dl" +cat "$dest/pre_run_bundle/pre-run.meta.json" diff --git a/.github/scripts/snapshot-export.sh b/.github/scripts/snapshot-export.sh new file mode 100755 index 0000000..47989f9 --- /dev/null +++ b/.github/scripts/snapshot-export.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Export $DATADIR as /jochemnet/$CLIENT// with its metadata and head block +# (benchmarkoor snapshot export). The head block is geth's, from the snapshot-tail artifact. +# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET +set -euo pipefail +blk=$SNAPSHOT_TAIL_DIR/head.json +meta=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json +head=$(( $(jq -r .result.number "$blk") )) + +jq -n --arg img "$SNAPSHOT_IMAGE" --arg release "$RELEASE" --arg eest "$EEST" --argjson base "$BASE" \ + --argjson head "$head" --arg hash "$(jq -r .result.hash "$blk")" --argjson ams "$AMSTERDAM_TIME" \ + --argjson gas "$SNAPSHOT_GAS_LIMIT" --argjson ramp "$(jq .payloads "$meta")" \ + --arg run "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" '{ + docker_image: $img, + shadowfork: {network: "jochemnet", base: $base, release: $release, eest: $eest, + head: $head, head_hash: $hash, gas_limit: $gas, ramp_blocks: $ramp, amsterdam_time: $ams, + filler: "geth", built_by: $run, + notes: "already on Amsterdam: a devnet must start on Gloas with amsterdamTime = amsterdam_time"}}' \ + > "$RUNNER_TEMP/snapshot/metadata.json" + +sudo -E "$RUNNER_TEMP/benchmarkoor" snapshot export --client "$CLIENT" --datadir "$DATADIR" \ + --network jochemnet --block "$head" --bucket "$BUCKET" \ + --head-block-file "$blk" --metadata-file "$RUNNER_TEMP/snapshot/metadata.json" diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml new file mode 100644 index 0000000..36ec350 --- /dev/null +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -0,0 +1,322 @@ +name: benchmarkoor-snapshot + +# Turns a jochemnet benchmarkoor-release into a shadowfork snapshot every client +# can boot without a pre-run of its own (msf-2): the release's pre-run applied to +# each client's jochemnet datadir, the gas limit walked down, the datadir +# compacted, and the result published to /jochemnet///. +# +# prepare release manifest → base and head blocks, context; bucket write probe +# geth replay the release's pre-run, ramp the gas limit down (the tail +# bundle), reopen at the head, compact, export +# replay every other client: release pre-run + tail, compact, export +# latest only once every client is up: /jochemnet//latest +# +# Each payload must come back VALID on every client, so every image has geth's +# state root, receipts and block access list at every block. Runs on the +# jochemnet hosts, whose schelk baseline holds the release's base snapshot; the +# scratch is recovered afterwards so the next benchmark starts from the baseline. +# +# Needs benchmarkoor with pre_runs base_bundle and fill: false (#349), +# `db compact` (#351) and `snapshot export` (#350): until they merge, dispatch +# with image "" and a git-ref carrying all three. +run-name: "benchmarkoor-snapshot · ${{ inputs.release }} · gas=${{ inputs.gas-limit }} · publish=${{ inputs.publish }}" + +on: + workflow_dispatch: + inputs: + release: + description: "jochemnet benchmarkoor-release whose pre-run the snapshot carries" + default: "eest-payloads-jochemnet-v1-amsterdam-stateful-e269bb44-20260916-124530" + gas-limit: + description: "Gas limit of the published head (the release's payloads run at 1 TGas)" + default: "200000000" + clients: + description: "Clients replaying geth's chain (JSON array); geth always runs" + default: '["nethermind", "besu", "reth", "erigon", "ethrex"]' + images: + description: "Client images (JSON): the releases the devnets run, which open and publish each datadir" + default: '{"geth": "ethereum/client-go:v1.17.7", "nethermind": "nethermind/nethermind:2.1.0", "besu": "hyperledger/besu:26.9.0", "reth": "ghcr.io/paradigmxyz/reth:v2.7.0", "erigon": "erigontech/erigon:v3.7.1", "ethrex": "ghcr.io/lambdaclass/ethrex:29.0.1"}' + compaction-image: + description: "RocksDB ldb image for nethermind/besu compaction" + default: "ghcr.io/ethpandaops/benchmarkoor-rocksdb-ldb:11.8.1" + bucket: + description: "Bucket published to as jochemnet///" + default: "ethpandaops-shadowfork-images" + publish: + description: "Export to the bucket and write latest (false: build, check and compact only)" + type: boolean + default: true + image: + description: "Benchmarkoor docker image. Empty builds it from git-ref." + default: "ghcr.io/ethpandaops/benchmarkoor:master" + git-ref: + description: "Benchmarkoor git ref, used only when image is empty" + default: "" + timeout-minutes: + description: "Per client job" + default: "2880" + +concurrency: + group: benchmarkoor-snapshot + cancel-in-progress: false + +env: + CONFIG_BASE: https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs + AWS_DEFAULT_REGION: auto + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + base: ${{ steps.release.outputs.base }} + head: ${{ steps.release.outputs.head }} + context: ${{ steps.release.outputs.context }} + subdir: ${{ steps.release.outputs.subdir }} + eest: ${{ steps.release.outputs.eest }} + ams: ${{ steps.release.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release + id: release + env: + TAG: ${{ inputs.release }} + run: | + m=$(curl -fsSL "https://github.com/${GITHUB_REPOSITORY}/releases/download/$TAG/manifest.json") + [ "$(jq -r .snapshot <<<"$m")" = jochemnet/v1 ] || { echo "::error::$TAG is not a jochemnet/v1 release"; exit 1; } + b=$(jq -ce '.assets[] | select(.client == "geth") | .build' <<<"$m") + base=$(jq -re .prerun_start_block <<<"$b") + # The hosts' schelk baseline is the snapshot named in the datadirs config; the + # release's pre-run has to start on it. + dir=$(sed -n 's/^ *GETH_SNAPSHOT_DIR: *//p' configs/datadirs/jochemnet/v1/global.yaml) + [ "${dir##*/}" = "$base" ] || { echo "::error::$TAG starts at $base, the hosts hold ${dir##*/}"; exit 1; } + # The release was built and is run on its context's schedule; so is the snapshot. + ctx="configs/contexts/$(jq -r .context <<<"$m")/$(jq -r .subdir <<<"$m")/global.yaml" + ams=$(sed -n 's/^ *AMSTERDAM_ACTIVATION_TS: *"\{0,1\}\([0-9]*\).*/\1/p' "$ctx") + [ -n "$ams" ] || { echo "::error::no AMSTERDAM_ACTIVATION_TS in $ctx"; exit 1; } + { + echo "ams=$ams" + echo "base=$base" + echo "head=$(jq -re .prerun_end_block <<<"$b")" + echo "context=$(jq -re .context <<<"$m")" + echo "subdir=$(jq -re .subdir <<<"$m")" + echo "eest=$(jq -re .eest_sha <<<"$m")" + } | tee -a "$GITHUB_OUTPUT" + + - name: Bucket write + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + BUCKET: ${{ inputs.bucket }} + run: | + key="jochemnet/.probe-${GITHUB_RUN_ID}" + if ! { echo probe | aws s3 cp - "s3://$BUCKET/$key" && aws s3 rm "s3://$BUCKET/$key"; }; then + echo "::error::the repo's S3 credentials cannot write $BUCKET"; exit 1 + fi + + geth: + needs: prepare + runs-on: + group: gasbenchmark + labels: + - benchmarkoor-v1-jochemnet-runner-geth + - benchmarkoor-v1-jochemnet-runner-benchmark-ci-3 + timeout-minutes: ${{ fromJSON(inputs.timeout-minutes) }} + env: + CLIENT: geth + SNAPSHOT_IMAGE: ${{ fromJSON(inputs.images).geth }} + SNAPSHOT_GAS_LIMIT: ${{ inputs.gas-limit }} + DATADIR: /schelk/snapshots/geth/jochemnet/${{ needs.prepare.outputs.base }} + AMSTERDAM_TIME: ${{ needs.prepare.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release pre-run + run: | + W=$RUNNER_TEMP/snapshot + { echo "W=$W"; echo "SNAPSHOT_BASE_BUNDLE=$W/release/pre_run_bundle"; echo "SNAPSHOT_TAIL_DIR=$W/tail"; } >> "$GITHUB_ENV" + .github/scripts/release-pre-run-bundle.sh "${{ inputs.release }}" "$W/release" + + - name: Replay the release, ramp the gas limit down + uses: ethpandaops/benchmarkoor@master + with: + mode: build + github-token: ${{ secrets.GITHUB_TOKEN }} + image: ${{ inputs.image }} + git-ref: ${{ inputs.git-ref }} + upload-artifacts: "false" + upload-build-artifacts: "false" + build-config-urls: >- + ${{ env.CONFIG_BASE }}/global.yaml, + ${{ env.CONFIG_BASE }}/datadirs/jochemnet/v1/global.yaml, + ${{ env.CONFIG_BASE }}/contexts/${{ needs.prepare.outputs.context }}/${{ needs.prepare.outputs.subdir }}/global.yaml, + ${{ env.CONFIG_BASE }}/snapshots/jochemnet/v1/builder.yaml + build-args: >- + --skip-state-actor-build + --skip-eest-payload-build + --limit-pre-runs-target=geth-ramp + + - name: Check the tail + env: + RELEASE_HEAD: ${{ needs.prepare.outputs.head }} + run: | + m=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json + cat "$m" + [ "$(jq -r .start_block_number "$m")" = "$RELEASE_HEAD" ] || { echo "::error::the ramp does not start on the release head $RELEASE_HEAD"; exit 1; } + last=$(tail -n 2 "$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.request" | head -n 1 | jq -r '.params[0].gasLimit') + [ $((last)) -eq "$SNAPSHOT_GAS_LIMIT" ] || { echo "::error::head gas limit $((last)), want $SNAPSHOT_GAS_LIMIT"; exit 1; } + + # The head's block is what a snapshot consumer anchors on, and reading it from a + # restarted geth proves the datadir reopens at the head: the journal written on + # stop holds only the last 128 ramp blocks (--cache.gc=0 flushed the rest). + - name: Reopen at the head + run: | + want=$(jq -r .end_block_hash "$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json") + docker run -d --name snapshot-geth-reopen --network host -v "$DATADIR:/data" "$SNAPSHOT_IMAGE" \ + --datadir /data --override.amsterdam="$AMSTERDAM_TIME" --nodiscover --maxpeers 0 --port 0 \ + --http --http.addr 127.0.0.1 --http.port 18545 --http.api eth --authrpc.port 18551 + for _ in $(seq 120); do + curl -sf -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}' \ + http://127.0.0.1:18545 > "$W/head.json" && break + sleep 5 + done + docker stop -t 600 snapshot-geth-reopen + code=$(docker inspect -f '{{.State.ExitCode}}' snapshot-geth-reopen) + docker logs --tail 30 snapshot-geth-reopen + docker rm snapshot-geth-reopen + [ "$code" = 0 ] || { echo "::error::geth exited $code"; exit 1; } + got=$(jq -r .result.hash "$W/head.json") + [ "$got" = "$want" ] || { echo "::error::geth reopened at $got, want $want"; exit 1; } + cp "$W/head.json" "$SNAPSHOT_TAIL_DIR/" + + - uses: actions/upload-artifact@v4 + with: + name: snapshot-tail + path: ${{ env.SNAPSHOT_TAIL_DIR }} + if-no-files-found: error + + - name: Compact + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" + + - name: Export + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + RELEASE: ${{ inputs.release }} + EEST: ${{ needs.prepare.outputs.eest }} + BASE: ${{ needs.prepare.outputs.base }} + BUCKET: ${{ inputs.bucket }} + run: .github/scripts/snapshot-export.sh + + - name: Clean up + if: always() + run: | + docker rm -f snapshot-geth-reopen >/dev/null 2>&1 || true + sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y + sudo rm -rf "$RUNNER_TEMP/snapshot" + + replay: + needs: [prepare, geth] + strategy: + fail-fast: false + matrix: + client: ${{ fromJSON(inputs.clients) }} + runs-on: + group: gasbenchmark + labels: + - benchmarkoor-v1-jochemnet-runner-${{ matrix.client }} + timeout-minutes: ${{ fromJSON(inputs.timeout-minutes) }} + env: + CLIENT: ${{ matrix.client }} + SNAPSHOT_IMAGE: ${{ fromJSON(inputs.images)[matrix.client] }} + SNAPSHOT_GAS_LIMIT: ${{ inputs.gas-limit }} + COMPACTION_IMAGE: ${{ inputs.compaction-image }} + DATADIR: /schelk/snapshots/${{ matrix.client }}/jochemnet/${{ needs.prepare.outputs.base }} + AMSTERDAM_TIME: ${{ needs.prepare.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release pre-run + run: | + W=$RUNNER_TEMP/snapshot + { echo "W=$W"; echo "SNAPSHOT_BASE_BUNDLE=$W/release/pre_run_bundle"; echo "SNAPSHOT_TAIL_DIR=$W/tail"; } >> "$GITHUB_ENV" + .github/scripts/release-pre-run-bundle.sh "${{ inputs.release }}" "$W/release" + + - uses: actions/download-artifact@v4 + with: + name: snapshot-tail + path: ${{ env.SNAPSHOT_TAIL_DIR }} + + - name: Replay the release and the ramp + uses: ethpandaops/benchmarkoor@master + with: + mode: build + github-token: ${{ secrets.GITHUB_TOKEN }} + image: ${{ inputs.image }} + git-ref: ${{ inputs.git-ref }} + upload-artifacts: "false" + upload-build-artifacts: "false" + build-config-urls: >- + ${{ env.CONFIG_BASE }}/global.yaml, + ${{ env.CONFIG_BASE }}/datadirs/jochemnet/v1/global.yaml, + ${{ env.CONFIG_BASE }}/contexts/${{ needs.prepare.outputs.context }}/${{ needs.prepare.outputs.subdir }}/global.yaml, + ${{ env.CONFIG_BASE }}/snapshots/jochemnet/v1/builder.yaml + build-args: >- + --skip-state-actor-build + --skip-eest-payload-build + --limit-pre-runs-target=${{ matrix.client }}-replay + + # ethrex has no compaction; reth (MDBX) is a logged no-op. + - name: Compact + if: matrix.client != 'ethrex' + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --compaction-image "$COMPACTION_IMAGE" + + - name: Export + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + RELEASE: ${{ inputs.release }} + EEST: ${{ needs.prepare.outputs.eest }} + BASE: ${{ needs.prepare.outputs.base }} + BUCKET: ${{ inputs.bucket }} + run: .github/scripts/snapshot-export.sh + + - name: Clean up + if: always() + run: | + sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y + sudo rm -rf "$RUNNER_TEMP/snapshot" + + # latest is written last and only once every client is up, so a consumer of it + # never meets a height some client lacks. + latest: + if: inputs.publish + needs: [prepare, geth, replay] + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + BUCKET: ${{ inputs.bucket }} + CLIENTS: ${{ inputs.clients }} + steps: + - uses: actions/download-artifact@v4 + with: + name: snapshot-tail + path: tail + + - name: Write latest + run: | + head=$(( $(jq -r .result.number tail/head.json) )) + clients=$(jq -r '["geth"] + . | unique | .[]' <<<"$CLIENTS") + for c in $clients; do + aws s3 ls "s3://$BUCKET/jochemnet/$c/$head/" | grep -q '\.tar\.zst$' || { echo "::error::no $c archive at $head"; exit 1; } + done + for c in $clients; do echo "$head" | aws s3 cp - "s3://$BUCKET/jochemnet/$c/latest"; done + echo "### jochemnet snapshot $head ($(jq -r .result.hash tail/head.json)): ${clients//$'\n'/ }" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/shadowfork-image-build.yaml b/.github/workflows/shadowfork-image-build.yaml deleted file mode 100644 index f5579c5..0000000 --- a/.github/workflows/shadowfork-image-build.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: shadowfork-image-build - -# Builds the geth half of a post-Gloas benchmark image: a fresh mainnet snapshot advanced by one -# pre-run (EIP-8282 contracts, Amsterdam, 24 KiB + 64 KiB receivers), its gas limit walked down, -# the fill flushed out of geth's journal and compacted. Uploads the geth image and the replay -# bundle; the other clients replay the bundle afterwards (shadowfork/README.md). `latest` is -# written only once every client is up. -run-name: "shadowfork-image-build · anchor=${{ inputs.anchor }} · receivers=${{ inputs.receivers }} · gas=${{ inputs.gas-target }}" - -on: - workflow_dispatch: - inputs: - anchor: - description: "Mainnet block of the snapshots.ethpandaops.io geth snapshot to start from (or latest)" - default: "latest" - receivers: - description: "Receivers per variant, at each code size (100000 covers benchmarks up to ~300 Mgas, 321000 needs 1 Ggas)" - default: "100000" - gas-target: - description: "Gas limit of the published head" - default: "200000000" - eest-repo: - description: "execution-specs repo for the fill" - default: "https://github.com/qu0b/execution-specs.git" - eest-ref: - description: "execution-specs ref: upstream forks/amsterdam + #3648 (fixture spill) + #3304 (deploy script) + the combined-sizes test" - default: "qu0b/bench-images-v3" - geth-image: - description: "geth release that fills and opens the image (the one the devnets run)" - default: "ethereum/client-go:v1.17.7" - benchmarkoor-image: - description: "benchmarkoor image" - default: "ghcr.io/ethpandaops/benchmarkoor:master" - runner: - description: "Runner (JSON). Needs ~2 TB free for a mainnet geth datadir plus the fill" - default: '{"group": "synctest", "labels": "Disk4TB"}' - publish: - description: "Upload the image and bundle (false: build and verify locally only)" - type: boolean - default: true - -concurrency: - group: shadowfork-image-build - cancel-in-progress: false - -jobs: - geth: - runs-on: ${{ fromJSON(inputs.runner) }} - timeout-minutes: 2880 - env: - GETH_IMAGE: ${{ inputs.geth-image }} - S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - uses: actions/checkout@v5 - - - name: Environment - run: | - echo "SF_WORK=${{ runner.temp }}/shadowfork" >> "$GITHUB_ENV" - for t in docker jq zstd curl python3; do command -v $t >/dev/null || { echo "::error::$t missing"; exit 1; }; done - command -v rclone >/dev/null || curl -fsSL https://rclone.org/install.sh | sudo bash - df -h "${{ runner.temp }}" - - - name: Bucket access - if: inputs.publish - run: | - source shadowfork/lib.sh - p="build-test/probe-${{ github.run_id }}" - if ! { echo probe | r2 rcat "r2:$BUCKET/$p" && r2 deletefile "r2:$BUCKET/$p"; }; then - echo "::error::the repo's S3 credentials cannot write $BUCKET"; exit 1 - fi - - - name: Snapshot and genesis - run: shadowfork/prepare.sh "${{ inputs.anchor }}" - - - name: Pre-run config - id: cfg - env: - RECEIVERS: ${{ inputs.receivers }} - EEST_REPO: ${{ inputs.eest-repo }} - EEST_REF: ${{ inputs.eest-ref }} - run: | - source shadowfork/lib.sh; source "$SF_WORK/anchor.env" - # The pre-run advances the datadir in place: schelk's CLI over a bind mount, no baseline. - { echo "SCHELK_DIR=$DATADIR"; echo "SCHELK_STATE=$SF_WORK/.schelk/state.json" - echo "BENCHMARKOOR_SCHELK_BIN=$PWD/shadowfork/inplace-schelk"; echo "TMPDIR=$SF_WORK/tmp"; } >> "$GITHUB_ENV" - mkdir -p "$SF_WORK/tmp" - SCHELK_DIR=$DATADIR SCHELK_STATE=$SF_WORK/.schelk/state.json shadowfork/inplace-schelk mount - { echo 'config<> "$GITHUB_OUTPUT" - - - name: Pre-run (fill) - uses: ethpandaops/benchmarkoor@master - env: - BENCHMARKOOR_BUILDER_RUN_TIMEOUT: 2400m - with: - mode: build - github-token: ${{ secrets.GITHUB_TOKEN }} - image: ${{ inputs.benchmarkoor-image }} - upload-artifacts: "false" - upload-build-artifacts: "false" - tmp-dir: ${{ env.SF_WORK }}/tmp - build-config: ${{ steps.cfg.outputs.config }} - build-args: --force - - - name: Ramp-down, journal flush, compaction, digest - env: - GAS_TARGET: ${{ inputs.gas-target }} - run: | - source shadowfork/lib.sh; source "$SF_WORK/anchor.env" - sudo touch "$DATADIR/.advanced" - sudo umount "$DATADIR" - shadowfork/finalize.sh - - - name: Publish - if: inputs.publish - run: shadowfork/publish.sh - - - name: Verify uploads - if: inputs.publish - run: | - source shadowfork/lib.sh - END=$(jq .end_block_number "$SF_WORK/bundle/pre_run_bundle/pre-run.meta.json") - shadowfork/verify.sh "mainnet/geth/$END/snapshot.tar.zst" "$SF_WORK/publish-geth.blocks" 3 - shadowfork/verify.sh "bundles/mainnet-$END/bundle.tar.zst" "$SF_WORK/publish-bundle.blocks" 1 - - - name: Summary - if: always() - run: | - m=$SF_WORK/bundle/pre_run_bundle/pre-run.meta.json - [ -f "$m" ] || exit 0 - { - echo "### shadowfork image (geth)" - echo '```json'; cat "$m"; echo '```' - [ -f "$SF_WORK/publish-geth.blocks" ] && echo "geth image: $(awk '$2=="total"{print $1}' "$SF_WORK/publish-geth.blocks") bytes" - echo - echo "Next: replay the bundle on the other clients, then metadata and \`latest\` (shadowfork/README.md)." - } >> "$GITHUB_STEP_SUMMARY" - - # The runner is shared (syncoor): never leave 1.5 TB behind. - - name: Clean up - if: always() - run: | - docker rm -f sf-geth >/dev/null 2>&1 || true - [ -z "${SF_WORK:-}" ] || { sudo umount "$SF_WORK/geth-datadir" 2>/dev/null || true; sudo rm -rf "$SF_WORK"; } diff --git a/README.md b/README.md index 2987db4..6a0071f 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ All workflows are `workflow_dispatch`-only and must be dispatched from the defau | `benchmarkoor-build-state-actor.yaml` | Build **only** the state-actor datadir (no payload fill). Has a `force` input to rebuild over an already-populated/partial datadir. | | `benchmarkoor-release.yaml` | Promote the filled payloads from a build run into a GitHub Release. | | `benchmarkoor-run.yaml` | Build the datadir, replay the payloads, and upload results to S3. | +| `benchmarkoor-snapshot.yaml` | Turn a jochemnet release into a shadowfork snapshot: every client's jochemnet datadir advanced by the release's pre-run, its gas limit walked down, compacted, and published as `/jochemnet///`. See below. | Common inputs: `clients` (JSON array), `snapshot`, `context`, `subdir`, `test-type`, and `instance-id`. `benchmarkoor-build-state-actor.yaml` takes only `snapshot` + `clients` (plus `force`), since the test context is irrelevant to a datadir-only build. @@ -93,6 +94,28 @@ Config merge order: - **build:** `global` → `datadirs//{global,builder}` → `contexts///{global, test-source..builder}` - **run:** `global` → resource-limits → `s3-upload` → `datadirs//{global,runner}` → `contexts///{global, test-source..runner, clients}` +### Shadowfork snapshots (`benchmarkoor-snapshot.yaml`) + +A devnet that shadowforks jochemnet (msf-2) boots these datadirs as they are, so it gets the +release's contracts without a pre-run of its own. Given a release tag: + +1. **geth** (the pinned jochemnet geth host) replays the release's pre-run onto its snapshot and + walks the gas limit down to `gas-limit` with empty blocks, recorded as the *tail* bundle + (`configs/snapshots/jochemnet/v1/builder.yaml`, target `geth-ramp`). It runs with + `--cache.gc=0`, so no block outside geth's 128 diff layers is left in the journal. A restarted + geth must come back at the tail's head; its head block is what every client publishes. +2. **every other client** replays the release's pre-run and then the tail onto its own snapshot. + Every payload must be VALID, so every image agrees with geth at every block. +3. Each datadir is **compacted** (`benchmarkoor db compact`; ldb per column family for + nethermind/besu, none for ethrex) so the release's contracts sit in the same DB tiers as old + state and read no faster than mainnet state would, then **exported** (`benchmarkoor snapshot export`). +4. `latest` is written last, once every client's archive is up. + +Runs on the jochemnet hosts, whose schelk baseline is the release's base snapshot (checked against +`configs/datadirs/jochemnet/v1/global.yaml`), and `schelk recover`s the scratch afterwards. +Publishing uses the repo's `S3_ENDPOINT_URL` / `AWS_*` secrets; the first job checks they can write +the bucket. Each host needs ~55 GB free for the release's pre-run bundle (50 GB uncompressed). + ## Dispatchoor The `dispatchoor/` directory holds generated job definitions for [dispatchoor](https://github.com/ethpandaops/dispatchoor) to trigger `benchmarkoor-run.yaml`. diff --git a/configs/snapshots/jochemnet/v1/builder.yaml b/configs/snapshots/jochemnet/v1/builder.yaml new file mode 100644 index 0000000..46c910b --- /dev/null +++ b/configs/snapshots/jochemnet/v1/builder.yaml @@ -0,0 +1,100 @@ +# Pre-runs that turn a jochemnet release into a shadowfork snapshot +# (benchmarkoor-snapshot.yaml). Loaded after configs/datadirs/jochemnet/v1/global.yaml +# (snapshot dirs, genesis files) and the release's own context global.yaml +# (AMSTERDAM_ACTIVATION_TS), so the schedule is the one the release was built on. +# +# The workflow sets SNAPSHOT_BASE_BUNDLE (the release's pre_run_bundle), +# SNAPSHOT_TAIL_DIR (where geth-ramp writes its bundle, the one the replays +# continue with), SNAPSHOT_GAS_LIMIT and SNAPSHOT_IMAGE, and builds one target +# per host with --limit-pre-runs-target. +builder: + pre_runs: + pull_policy: always + config: + fork: amsterdam + datadir_method: schelk + targets: + # geth continues the release's pre-run and walks the gas limit down from the + # payloads' 1 TGas with empty blocks (~1/1024 per block, ~8.7k blocks to + # 200M). --cache.gc=0 leaves geth no write buffer, so every block that + # leaves the 128 diff layers is flushed to pebble: the journal holds only + # the last 128 ramp blocks, never the release's contracts. + - name: geth-ramp + filler_client: geth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${GETH_SNAPSHOT_DIR} + bundle_dir: ${SNAPSHOT_TAIL_DIR} + genesis: ${GETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + fill: false + gas_limit: ${SNAPSHOT_GAS_LIMIT} + gas_bump_max_blocks: 10000 + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + - --cache.gc=0 + + # Every other client replays the release, then the ramp, onto its own + # jochemnet snapshot. + # Each payload must come back VALID, so each client re-derives geth's state + # root, receipts and block access list at every block. + - name: nethermind-replay + filler_client: nethermind + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${NETHERMIND_SNAPSHOT_DIR} + genesis: ${NETHERMIND_CHAINSPEC} + genesis_eip_override: + timestamp: ${AMSTERDAM_ACTIVATION_TS} + eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + # The snapshot dir is the chain column dir (datadirs/jochemnet/v1/global.yaml). + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + - --FlatDb.MinReorgDepth=1024 + - --FlatDb.MaxReorgDepth=1024 + + - name: besu-replay + filler_client: besu + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${BESU_SNAPSHOT_DIR} + genesis: ${BESU_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + + - name: reth-replay + filler_client: reth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${RETH_SNAPSHOT_DIR} + genesis: ${RETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + + - name: erigon-replay + filler_client: erigon + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${ERIGON_SNAPSHOT_DIR} + genesis: ${ERIGON_GENESIS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + + - name: ethrex-replay + filler_client: ethrex + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${ETHREX_SNAPSHOT_DIR} + genesis: ${ETHREX_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --skip-genesis-validation diff --git a/shadowfork/.gitignore b/shadowfork/.gitignore deleted file mode 100644 index c18dd8d..0000000 --- a/shadowfork/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/shadowfork/README.md b/shadowfork/README.md deleted file mode 100644 index c5aa977..0000000 --- a/shadowfork/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Shadowfork benchmark images - -Post-Gloas mainnet datadirs for every EL, ready for a mainnet shadowfork (msf-2) to boot and -benchmark without a prefill of its own. Published to `ethpandaops-shadowfork-images`, read at -`https://shadowfork-images.ethpandaops.io/mainnet///snapshot.tar.zst` with the -usual `_snapshot_eth_getBlockByNumber.json` / `_snapshot_metadata.json` / `latest` layout, so -ethereum-package's `network_sync_base_url` works unchanged. - -An image is a fresh `snapshots.ethpandaops.io` mainnet snapshot advanced by one pre-run that: - -1. funds the 25,000-account pool and buildoor's wallet, and deploys the EIP-8282 contracts in two - **Osaka** blocks (a strict client rejects every Amsterdam block without them); -2. crosses **Amsterdam** 3 s after the snapshot head, then ramps the gas limit to 2 G; -3. deploys `test_account_access`'s receivers at **24 KiB and 64 KiB**: SAME_MAX, DIFF_MAX and - JUMPDEST at both sizes, MINIMAL once, plus the 7702 delegations, in one test - (`test_deploy_existing_contracts_all_sizes`). 100k per variant covers benchmarks up to ~300 Mgas, - 1 Ggas needs ~321k; -4. walks the gas limit back down to the target (200 M) with empty blocks. - -geth fills; every other client **replays** the recorded bundle, so all images are one chain with one -head hash, and a client is only published if its digest (hash, stateRoot, gasUsed, BAL hash at ~200 -heights) is identical to geth's. - -## A clean image: nothing recent may be faster to read than old mainnet state - -Benchmarks against a prefilled snapshot found the contracts deployed after the snapshot in higher -DB tiers, and for some clients still warm in the journal, so they read faster than real mainnet -state would. Each client is cleaned before it is packed: - -| client | journal / hot layer | compaction | -|---|---|---| -| geth | path scheme keeps 128 diff layers plus an unflushed write buffer (up to 256 MiB) and writes both to `triedb/merkle.journal` on a clean stop, which the next start reads back into memory. The 2026-10-08 image journaled **1,354** blocks: the last ~1,200 fill blocks never reached pebble. The ramp-down runs with `--cache.gc=0` (a 0-byte buffer), so every block that leaves the diff layers is flushed, and `finalize.sh` fails unless the journal holds exactly the last 128 (empty) blocks | `geth db compact` | -| erigon | `seg retire` | `erigon db compact` | -| nethermind, besu | none | RocksDB `ldb compact` per column family, offline (neither ships a compaction command) | -| reth | none: MDBX, persisted every block | none needed | - -The geth mechanism was verified on a dev chain (v1.17.7): 176 blocks of state journaled as -`layers=176`; restarted with `--cache.gc=0` (`buffer=0.00B`), ~150 later blocks left `layers=128`. - -## Running it - -`shadowfork-image-build` (workflow_dispatch) does the geth half on a `synctest` `Disk4TB` runner: -snapshot, pre-run, ramp-down + flush + compaction, reference digest, then uploads -`mainnet/geth//` and `bundles/mainnet-/bundle.tar.zst` (payloads, genesis, anchor, -`digest-geth.txt`, head block). It never writes `latest`. - -The other clients are replayed separately, one machine each, from that bundle: download the -client's `snapshots.ethpandaops.io/mainnet//` snapshot, replay every payload -(each must be VALID), check that it reopens at the bundle's head, clean it as above, compare its -digest against `digest-geth.txt`, then pack and upload. Once all five are up and verified, write -each client's `_snapshot_metadata.json` and head block, and `latest` last. - -Requirements learned the hard way: -- Open each datadir with the **release** that wrote the snapshot and that the devnets run, not a - devnet build: the glamsterdam-devnet-8 images trailed by a major version (nethermind 1.40 vs 2.1). -- The fill must not hold its fixture in memory: each 64 KiB deploy's code is also in its block - access list, so the bundle is ~55 GB and an in-memory fixture passed 200 GiB. The default EEST - ref (`qu0b/execution-specs@qu0b/bench-images-v3`) is upstream `forks/amsterdam` plus the two open - upstream PRs this needs, ethereum/execution-specs#3648 (spill payload lists to disk) and #3304 - (the deployment script), plus the combined-sizes test and JUMPDEST at 24 KiB. -- Every long step is its own process: no host may auto-upgrade or restart services mid-build - (unattended-upgrades restarted a unit and killed a 1 TB upload with it). - -## Files - -| file | does | -|---|---| -| `prepare.sh` | download the geth snapshot (parallel ranged GETs), genesis with Amsterdam at head + 3 s | -| `prerun-config.sh` | the `benchmarkoor build` pre-run config (predeploy block lifted from the devnet-8 prefill) | -| `inplace-schelk` | schelk's CLI over a bind-mounted directory, so the pre-run advances the datadir in place | -| `rampdown.py` | empty blocks down to the gas target, appended to the bundle | -| `finalize.sh` | ramp-down with a 0-byte buffer, journal check, compaction, reopen, reference digest | -| `publish.sh`, `blockhash.py`, `verify.sh` | stream to the bucket hashing every GiB; re-read random GiBs | diff --git a/shadowfork/blockhash.py b/shadowfork/blockhash.py deleted file mode 100755 index 599f8f8..0000000 --- a/shadowfork/blockhash.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -"""sha256 of every 1 GiB block of stdin: ' ' per block, then ' total'.""" -import hashlib, sys - -BLOCK = 1 << 30 -i = total = 0 -h, n = hashlib.sha256(), 0 -for chunk in iter(lambda: sys.stdin.buffer.read(1 << 22), b""): - total += len(chunk) - while chunk: - take = chunk[: BLOCK - n] - h.update(take); n += len(take); chunk = chunk[len(take):] - if n == BLOCK: - print(i, h.hexdigest(), flush=True); i += 1; h, n = hashlib.sha256(), 0 -if n: - print(i, h.hexdigest(), flush=True) -print(total, "total", flush=True) diff --git a/shadowfork/finalize.sh b/shadowfork/finalize.sh deleted file mode 100755 index e63d679..0000000 --- a/shadowfork/finalize.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# After the pre-run: walk the gas limit down, push the fill out of geth's journal, compact, and -# write the reference digest the replaying clients are checked against. -# GAS_TARGET=200000000 shadowfork/finalize.sh -# -# Why each step (shadowfork/README.md, "A clean image"): -# - geth's path scheme keeps the last 128 blocks as in-memory diff layers and everything older but -# unflushed in a write buffer (up to 256 MiB); a clean stop writes both to triedb/merkle.journal, -# and the next start reads them back into memory. The pre-run's last ~1,200 blocks of deployed -# contracts sat there (2026-10-08), so a benchmark found them in RAM rather than on disk. -# --cache.gc=0 makes the buffer 0 bytes: every block that leaves the diff layers is flushed. -# - Freshly written keys sit in the top LSM levels while mainnet's state is in the bottom ones; -# `geth db compact` rewrites everything to the bottom level. -source "$(dirname "$0")/lib.sh" -source "$WORK/anchor.env" -GAS_TARGET=${GAS_TARGET:-200000000} -B=$WORK/bundle/pre_run_bundle -[ -f "$B/pre-run.meta.json" ] || die "no bundle at $B" -lines=$(wc -l < "$B/pre-run.request"); payloads=$(jq .payloads "$B/pre-run.meta.json") -[ "$lines" -eq $((payloads * 2)) ] || die "bundle has $lines lines for $payloads payloads" - -log "=== ramp-down to $GAS_TARGET with a 0-byte write buffer ===" -boot_geth --cache.gc=0 --miner.gaslimit="$GAS_TARGET" -docker logs sf-geth 2>&1 | grep -q 'buffer=0.00B' || die "geth did not start with a 0-byte write buffer" -python3 "$SF_DIR/rampdown.py" "http://127.0.0.1:$P_RPC" "http://127.0.0.1:$P_ENGINE" "$WORK/jwtsecret" "$WORK/bundle" "$GAS_TARGET" -stop_geth rampdown -# The journal may hold the 128 diff layers of the last (empty) blocks and nothing else. -grep -E 'Persisting dirty state' "$WORK/geth-rampdown.log" | tail -1 -grep -E 'Persisting dirty state.*layers=128$' "$WORK/geth-rampdown.log" >/dev/null \ - || die "geth journaled more than the 128 diff layers: the fill is still in memory" - -log "=== geth db compact ===" -docker run --rm -v "$DATADIR:/data" "$GETH_IMAGE" db compact --datadir=/data 2>&1 | tail -5 - -log "=== reopen and reference digest ===" -END=$(jq .end_block_number "$B/pre-run.meta.json"); END_HASH=$(jq -r .end_block_hash "$B/pre-run.meta.json") -boot_geth -h=$(rpc eth_getBlockByNumber '["latest",false]') -[ "$(( $(jq -r .result.number <<<"$h") ))" = "$END" ] && [ "$(jq -r .result.hash <<<"$h")" = "$END_HASH" ] \ - || die "geth reopens at $(jq -c '.result | [.number, .hash]' <<<"$h"), not $END $END_HASH" -# The same heights lib/replay.py digests on a replaying client: first payload, every n/200th. -start=$(head -1 "$B/pre-run.request" | jq -r '.params[0].blockNumber'); start=$((start)) -n=$(jq .payloads "$B/pre-run.meta.json"); step=$(( n / 200 > 0 ? n / 200 : 1 )) -for ((b = start; b < start + n; b += step)); do - r=$(rpc eth_getBlockByNumber "[\"$(printf 0x%x $b)\",false]") - jq -r --argjson b "$b" '"\($b) \(.result.hash) \(.result.stateRoot) \(.result.gasUsed) \(.result.blockAccessListHash // "none")"' <<<"$r" -done > "$WORK/digest-geth.txt" -rpc eth_getBlockByNumber "[\"$(printf 0x%x "$END")\",true]" > "$WORK/_snapshot_eth_getBlockByNumber.json" -stop_geth final -log "image head $END $END_HASH, gas limit $(( $(jq -r .result.gasLimit "$WORK/_snapshot_eth_getBlockByNumber.json") )), $(wc -l < "$WORK/digest-geth.txt") digest heights" diff --git a/shadowfork/inplace-schelk b/shadowfork/inplace-schelk deleted file mode 100755 index f2a0d78..0000000 --- a/shadowfork/inplace-schelk +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# schelk's CLI, as benchmarkoor drives it (pkg/datadir/schelk.go), on a plain directory: the -# pre-run advances it in place and nothing can roll it back. For a disposable builder whose disk -# cannot hold a copy (build/README.md); tools/zfs-schelk keeps a pristine @baseline instead. -# -# SCHELK_DIR the datadir (required); bind-mounted onto itself so it is in /proc/mounts -# SCHELK_STATE state file benchmarkoor reads (default /var/lib/schelk/state.json) -# -# restore/recover before a pre-run are no-ops on a pristine dir. Once 01-prerun.sh has marked it -# .advanced they fail: a rollback that cannot happen must not report success. -set -euo pipefail -DIR=${SCHELK_DIR:?SCHELK_DIR is not set} -STATE=${SCHELK_STATE:-/var/lib/schelk/state.json} -die() { echo "inplace-schelk: $*" >&2; exit 1; } - -mkdir -p "$(dirname "$STATE")" -exec 9>"${STATE%.json}.lock" -# benchmarkoor retries on exactly this text while another schelk holds the lock. -flock -n 9 || die "Another schelk process is already running" -[ -d "$DIR" ] || die "no directory $DIR" -mountpoint -q "$DIR" || sudo mount --bind "$DIR" "$DIR" - -case "${1:-}" in - init|mount|promote) ;; - restore|recover|full-recover) - [ ! -e "$DIR/.advanced" ] || die "$DIR was advanced in place and has no baseline to $1 to" ;; - *) die "unsupported verb '${1:-}' (init|restore|recover|full-recover|mount|promote)" ;; -esac -printf '{"mount_point":"%s","is_mounted":true,"backend":"inplace"}\n' "$DIR" > "$STATE" diff --git a/shadowfork/lib.sh b/shadowfork/lib.sh deleted file mode 100755 index ff69d8a..0000000 --- a/shadowfork/lib.sh +++ /dev/null @@ -1,39 +0,0 @@ -# shellcheck shell=bash disable=SC2034 # sourced: the settings are used by the scripts that source it -# Shared settings for the shadowfork image build (see shadowfork/README.md). -set -euo pipefail -SF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -WORK=${SF_WORK:?SF_WORK must point at a directory on the big disk} -SNAP=${SF_SNAPSHOTS:-https://snapshots.ethpandaops.io} -BUCKET=${SF_BUCKET:-ethpandaops-shadowfork-images} -PUBLIC=${SF_PUBLIC:-https://shadowfork-images.ethpandaops.io} -# The image is opened by the release that wrote the snapshot and that the devnets run. -GETH_IMAGE=${GETH_IMAGE:-ethereum/client-go:v1.17.7} -P_RPC=19545 P_ENGINE=19551 -mkdir -p "$WORK" - -log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } -die() { log "ERROR: $*" >&2; exit 1; } -rpc() { # rpc [params-json] - curl -sf --max-time 60 -X POST -H 'content-type: application/json' \ - --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$1\",\"params\":${2:-[]}}" "http://127.0.0.1:$P_RPC" -} -r2() { # rclone against the images bucket with the repo's S3 credentials - RCLONE_CONFIG_R2_TYPE=s3 RCLONE_CONFIG_R2_PROVIDER=Cloudflare RCLONE_CONFIG_R2_ENDPOINT=${S3_ENDPOINT_URL:?} \ - RCLONE_CONFIG_R2_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:?} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:?} \ - rclone "$@" -} -boot_geth() { # boot_geth : the image datadir on the fixed ports, no peers - source "$WORK/anchor.env" - docker rm -f sf-geth >/dev/null 2>&1 || true - docker run -d --name sf-geth --network host -v "$DATADIR:/data" -v "$GENESIS:/genesis.json:ro" \ - -v "$WORK/jwtsecret:/jwtsecret:ro" "$GETH_IMAGE" \ - --datadir=/data --override.genesis=/genesis.json --override.amsterdam="$AMS_TS" \ - --http --http.addr=127.0.0.1 --http.port=$P_RPC --http.api=eth,debug,testing \ - --authrpc.addr=127.0.0.1 --authrpc.port=$P_ENGINE --authrpc.jwtsecret=/jwtsecret \ - --syncmode=full --maxpeers=0 --nodiscover "$@" >/dev/null - for _ in $(seq 120); do rpc eth_blockNumber >/dev/null 2>&1 && return 0; sleep 5; done - docker logs --tail 20 sf-geth; die "geth did not open its RPC" -} -stop_geth() { # graceful: a killed geth never writes its trie journal and reopens short of its head - docker stop -t 1800 sf-geth >/dev/null; docker logs sf-geth > "$WORK/geth-$1.log" 2>&1; docker rm sf-geth >/dev/null -} diff --git a/shadowfork/pfetch.sh b/shadowfork/pfetch.sh deleted file mode 100755 index 8c00217..0000000 --- a/shadowfork/pfetch.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -# pfetch.sh [workers] [chunk-bytes] : write to stdout, fetched as parallel -# ranged GETs and emitted in order. One stream from R2 via Cloudflare tops out ~95 MB/s; -# 16 ranges measured 239-349 MB/s. POSIX sh + curl, so it runs in ethereum-package's -# alpine downloader: `pfetch.sh "$URL" | tar -I zstd -xf - -C "$DIR"`. -# At most 2 x workers chunks are staged in $TMPDIR; a chunk is retried until complete. -set -eu -URL=$1; P=${2:-16}; C=${3:-67108864} -TOTAL=$(curl -sfIL "$URL" | tr -d '\r' | awk 'tolower($1)=="content-length:"{n=$2} END{print n}') -[ -n "$TOTAL" ] || { echo "pfetch: no Content-Length for $URL" >&2; exit 1; } -[ -z "${PFETCH_BYTES:-}" ] || [ "$PFETCH_BYTES" -ge "$TOTAL" ] || TOTAL=$PFETCH_BYTES # testing: a prefix only -N=$(( (TOTAL + C - 1) / C )) -T=$(mktemp -d); echo 0 > "$T/emitted" -PIDS="" # our workers only: `kill 0` would also hit the tar we are piped into -trap 'kill $PIDS 2>/dev/null || true; rm -rf "$T"' EXIT INT TERM - -fetch() { # fetch : until the chunk file holds exactly its range - s=$(( $1 * C )); e=$(( s + C - 1 )); [ "$e" -lt "$TOTAL" ] || e=$(( TOTAL - 1 )) - want=$(( e - s + 1 )); tries=0 - while :; do - # --max-filesize: a cold, cache-eligible object can answer a Range with 200 and the - # whole body (measured on snapshots.ethpandaops.io); refuse it before a byte lands. - curl -sfL --connect-timeout 20 --speed-limit 1024 --speed-time 60 --max-filesize "$want" \ - -r "$s-$e" -o "$T/$1.part" "$URL" || true - [ "$(wc -c 2>/dev/null < "$T/$1.part" || echo 0)" -eq "$want" ] && { mv "$T/$1.part" "$T/$1"; return; } - tries=$(( tries + 1 )); [ "$tries" -lt 50 ] || { echo "pfetch: chunk $1 failed $tries times" >&2; touch "$T/failed"; return 1; } - sleep 2 - done -} - -k=0 -while [ "$k" -lt "$P" ]; do - { ( i=$k - while [ "$i" -lt "$N" ]; do - while [ $(( i - $(cat "$T/emitted") )) -ge $(( 2 * P )) ]; do sleep 0.1; done - fetch "$i" || exit 1 - i=$(( i + P )) - done ) || touch "$T/failed"; } & # a worker that dies must fail the stream, not hang it - PIDS="$PIDS $!"; k=$(( k + 1 )) -done - -i=0 -while [ "$i" -lt "$N" ]; do - while [ ! -f "$T/$i" ]; do [ ! -f "$T/failed" ] || exit 1; sleep 0.05; done - cat "$T/$i"; rm -f "$T/$i"; i=$(( i + 1 )) - echo "$i" > "$T/emitted.tmp"; mv "$T/emitted.tmp" "$T/emitted" # atomic: a torn read kills a worker -done -wait diff --git a/shadowfork/prepare.sh b/shadowfork/prepare.sh deleted file mode 100755 index 388db17..0000000 --- a/shadowfork/prepare.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Download the anchor mainnet geth snapshot and write the pre-run's genesis. -# shadowfork/prepare.sh [block|latest] -# Amsterdam is scheduled 3 s after the snapshot head: the pre-run's two funding/predeploy blocks -# are Osaka (the EIP-8282 contracts must exist before the fork), everything after is Amsterdam. -source "$(dirname "$0")/lib.sh" -BLOCK=${1:-latest} -[ "$BLOCK" != latest ] || BLOCK=$(curl -sf "$SNAP/mainnet/geth/latest" | tr -d '[:space:]') -[[ "$BLOCK" =~ ^[0-9]+$ ]] || die "no anchor block ($BLOCK)" -DATADIR=$WORK/geth-datadir -URL=$SNAP/mainnet/geth/$BLOCK/snapshot.tar.zst - -# The pre-run advances the datadir in place, so the disk needs the extracted snapshot plus the -# fill's delta, its fixtures and the bundle (55 GB at 100k receivers per variant). -need=$(( $(curl -sf "$SNAP/mainnet/geth/$BLOCK/_snapshot_metadata.json" | jq -r .data_size_bytes) + 400 * 1024**3 )) -free=$(( $(df -B1 --output=avail "$WORK" | tail -1) )) -[ "$free" -gt "$need" ] || die "$WORK has $((free / 1024**3)) GiB free, needs $((need / 1024**3))" - -if [ ! -f "$DATADIR/.complete" ]; then - sudo rm -rf "$DATADIR"; mkdir -p "$DATADIR/geth" - log "downloading mainnet/geth/$BLOCK ($(curl -sfI "$URL" | tr -d '\r' | awk 'tolower($1)=="content-length:"{printf "%.0f GiB", $2/2^30}'))" - # The published snapshot has no geth/ prefix; geth wants /geth/chaindata. - "$SF_DIR/pfetch.sh" "$URL" "${PFETCH_WORKERS:-16}" | zstd -d --stdout | sudo tar -xf - -C "$DATADIR/geth" \ - || die "download/extract failed" - sudo touch "$DATADIR/.complete" -fi - -HEAD=$(curl -sf "$SNAP/mainnet/geth/$BLOCK/_snapshot_eth_getBlockByNumber.json") -HEAD_TS=$(( $(jq -r '.result.timestamp' <<<"$HEAD") )) -AMS_TS=$((HEAD_TS + 3)) -GENESIS=$WORK/genesis.json -# Only forks not yet active at the head may move: moving an active one makes geth rewind. -curl -sf https://raw.githubusercontent.com/eth-clients/mainnet/refs/heads/main/metadata/genesis.json \ - | jq --argjson h "$HEAD_TS" --argjson a "$AMS_TS" ' - .config |= with_entries(if (.key|test("^(osaka|bpo[0-9]+)Time$")) and (.value > $h) then .value = $h + 1 else . end) - | .config.amsterdamTime = $a' > "$GENESIS" -[ "$(jq -r '[.config | to_entries[] | select(.key|test("Time$")) | .value] | max' "$GENESIS")" = "$AMS_TS" ] \ - || die "a parent fork is scheduled after amsterdam; this anchor cannot cross it" -[ -f "$WORK/jwtsecret" ] || openssl rand -hex 32 > "$WORK/jwtsecret" -cat > "$WORK/anchor.env" <.blocks (sha256 of every GiB of the uploaded stream) for verify.sh. -# Never writes `latest`: that waits until every client's image is up (shadowfork/README.md). -source "$(dirname "$0")/lib.sh" -source "$WORK/anchor.env" -B=$WORK/bundle/pre_run_bundle -END=$(jq .end_block_number "$B/pre-run.meta.json") -up() { # up : stdin -> zstd -> hashed -> bucket - zstd -6 -T0 -q | tee >(python3 "$SF_DIR/blockhash.py" > "$WORK/publish-$1.blocks") \ - | r2 rcat --s3-chunk-size 256M --s3-upload-concurrency 8 "r2:$BUCKET/$2" - sleep 5 # the process substitution may still be flushing - grep -q total "$WORK/publish-$1.blocks" || die "$1: upload stream incomplete" -} -docker ps --format '{{.Names}}' | grep -qx sf-geth && die "geth is still running" - -log "=== geth image -> mainnet/geth/$END ===" -# The geth/ prefix is load-bearing: /geth/triedb holds the state geth reopens with. -sudo tar --sort=name -C "$DATADIR" --exclude=geth/nodekey --exclude=geth/LOCK --exclude=geth/nodes \ - --exclude='geth/_snapshot_*' -cf - geth | up geth "mainnet/geth/$END/snapshot.tar.zst" -r2 copyto "$WORK/_snapshot_eth_getBlockByNumber.json" "r2:$BUCKET/mainnet/geth/$END/_snapshot_eth_getBlockByNumber.json" - -log "=== bundle -> bundles/mainnet-$END ===" -# Everything a replaying client needs: the payloads, the genesis they were built against, the -# anchor, and geth's digest to compare its own against. -tar --sort=name -C "$WORK" -cf - anchor.env genesis.json digest-geth.txt _snapshot_eth_getBlockByNumber.json \ - bundle/pre_run_bundle/pre-run.request bundle/pre_run_bundle/pre-run.meta.json | up bundle "bundles/mainnet-$END/bundle.tar.zst" -log "published head $END: geth image $(awk '$2=="total"{print $1}' "$WORK/publish-geth.blocks") B, bundle $(awk '$2=="total"{print $1}' "$WORK/publish-bundle.blocks") B" diff --git a/shadowfork/rampdown.py b/shadowfork/rampdown.py deleted file mode 100755 index db0f917..0000000 --- a/shadowfork/rampdown.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -"""Append empty blocks to the image until its gas limit is the target, and record them in the bundle. - -The pre-run leaves the head at its 2 G bump target; a devnet that wants 200 M would otherwise -spend ~2,400 blocks walking down 1/1024 per block. The blocks are built the way benchmarkoor -builds its own (testing_buildBlockV1, then engine_newPayloadV5 + engine_forkchoiceUpdatedV3), -with geth started at --miner.gaslimit=, and appended to pre-run.request so every replayed -client reaches the same head. At least MIN_BLOCKS are built even once the target is reached: they -push the fill out of geth's 128 in-memory diff layers. - -Usage: rampdown.py -""" -import base64, hashlib, hmac, json, sys, time, urllib.request - -rpc_url, engine_url, jwt_path, bdir, target = sys.argv[1:6] -target = int(target) -MIN_BLOCKS = 256 -EXTRA_DATA = "0x62656e63686d61726b6f6f72" # "benchmarkoor", as on its own blocks -secret = bytes.fromhex(open(jwt_path).read().strip().removeprefix("0x")) - -def jwt(): - b = lambda x: base64.urlsafe_b64encode(x).rstrip(b"=") - h, p = b(b'{"alg":"HS256","typ":"JWT"}'), b(json.dumps({"iat": int(time.time())}).encode()) - return (h + b"." + p + b"." + b(hmac.new(secret, h + b"." + p, hashlib.sha256).digest())).decode() - -def call(url, method, params, auth=False): - hdr = {"Content-Type": "application/json"} - if auth: - hdr["Authorization"] = "Bearer " + jwt() - body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode() - r = json.load(urllib.request.urlopen(urllib.request.Request(url, data=body, headers=hdr), timeout=120)) - if "error" in r: - sys.exit(f"{method}: {r['error']}") - return r["result"] - -req_path, meta_path = f"{bdir}/pre_run_bundle/pre-run.request", f"{bdir}/pre_run_bundle/pre-run.meta.json" -meta = json.load(open(meta_path)) -head = call(rpc_url, "eth_getBlockByNumber", ["latest", False]) -if int(head["number"], 16) != meta["end_block_number"] or head["hash"] != meta["end_block_hash"]: - sys.exit(f"geth head {int(head['number'], 16)} is not the bundle end {meta['end_block_number']}") - -i, built, prev_gl = meta["payloads"], 0, int(head["gasLimit"], 16) -with open(req_path, "a") as f: - while built < MIN_BLOCKS or int(head["gasLimit"], 16) > target: - attrs = {"timestamp": hex(int(head["timestamp"], 16) + 1), "prevRandao": head["hash"], - "suggestedFeeRecipient": "0x" + "00" * 20, "withdrawals": [], - "parentBeaconBlockRoot": head["hash"]} - # The image is past Amsterdam (V5, slot numbers); an Osaka chain takes V4 (used by the test). - amsterdam = head.get("slotNumber") is not None - if amsterdam: - attrs["slotNumber"] = hex(int(head["slotNumber"], 16) + 1) - method = "engine_newPayloadV5" if amsterdam else "engine_newPayloadV4" - res = call(rpc_url, "testing_buildBlockV1", [head["hash"], attrs, [], EXTRA_DATA]) - payload = res.get("executionPayload", res) - params = [payload, [], head["hash"], res.get("executionRequests") or []] - st = call(engine_url, method, params, auth=True) - if st["status"] != "VALID": - sys.exit(f"block {int(payload['blockNumber'], 16)}: newPayload {st}") - fc = {"finalizedBlockHash": payload["blockHash"], "headBlockHash": payload["blockHash"], - "safeBlockHash": payload["blockHash"]} - call(engine_url, "engine_forkchoiceUpdatedV3", [fc, None], auth=True) - i += 1 - f.write(json.dumps({"id": i, "jsonrpc": "2.0", "method": method, "params": params}, - separators=(",", ":")) + "\n") - f.write(json.dumps({"id": i, "jsonrpc": "2.0", "method": "engine_forkchoiceUpdatedV3", "params": [fc, None]}, - separators=(",", ":")) + "\n") - head = call(rpc_url, "eth_getBlockByNumber", ["latest", False]) - gl = int(head["gasLimit"], 16) - if built == 0 and gl >= prev_gl and gl > target: - sys.exit(f"gas limit did not fall ({prev_gl} -> {gl}): is geth running with --miner.gaslimit={target}?") - built += 1 - if built % 500 == 0: - print(f" {built} blocks, gas limit {gl}", flush=True) - -meta.update(payloads=i, end_block_number=int(head["number"], 16), end_block_hash=head["hash"], - ramp_down_blocks=built, end_gas_limit=int(head["gasLimit"], 16)) -json.dump(meta, open(meta_path, "w"), indent=2) -print(f"ramp-down: {built} blocks to gas limit {int(head['gasLimit'], 16)}, head {meta['end_block_number']} {head['hash']}") diff --git a/shadowfork/verify.sh b/shadowfork/verify.sh deleted file mode 100755 index 2735d93..0000000 --- a/shadowfork/verify.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Check an uploaded object against the block hashes publish.sh took of the stream it sent: its size, -# then N random 1 GiB blocks. Images are read through the public host as consumers read them (each -# Range must be a 206); bundles/ is not public (WAF), so it is read with the bucket credentials. -# shadowfork/verify.sh [N=3] -source "$(dirname "$0")/lib.sh" -obj=${1:?object} f=${2:?blocks file} n=${3:-3} -want=$(awk '$2=="total"{print $1}' "$f") -if [[ $obj == mainnet/* ]]; then - got=$(curl -sfI "$PUBLIC/$obj" | tr -d '\r' | awk 'tolower($1)=="content-length:"{print $2}') -else - got=$(r2 size --json "r2:$BUCKET/$obj" | jq .bytes) -fi -[ "$got" = "$want" ] || die "$obj: size $got, uploaded $want" -blocks=$(grep -vc total "$f") -for i in $(shuf -i 0-$((blocks - 1)) -n "$(( n < blocks ? n : blocks ))"); do - s=$((i << 30)); len=$(( (1 << 30) )); [ $((s + len)) -le "$want" ] || len=$((want - s)) - if [[ $obj == mainnet/* ]]; then - code=$(curl -s -o "$WORK/vb.$i" -w '%{http_code}' -r "$s-$((s + len - 1))" "$PUBLIC/$obj") - [ "$code" = 206 ] || die "$obj block $i: HTTP $code" - else - r2 cat --offset "$s" --count "$len" "r2:$BUCKET/$obj" > "$WORK/vb.$i" - fi - [ "$(sha256sum < "$WORK/vb.$i" | cut -d' ' -f1)" = "$(awk -v i="$i" '$1==i{print $2}' "$f")" ] \ - || die "$obj block $i: hash mismatch" - rm -f "$WORK/vb.$i"; log "$obj block $i OK" -done -log "$obj: $want bytes, size and random blocks match" From 6306e64eeddd0b28b2bac761e5dd63da44a0c445 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 16:53:46 +0200 Subject: [PATCH 04/14] benchmarkoor-snapshot: SNAPSHOT_PREFIX for a scratch export; full-tx head block snapshot export publishes the head block as-is, and its contract is the full-transaction eth_getBlockByNumber response. --- .github/scripts/snapshot-export.sh | 5 +++-- .github/workflows/benchmarkoor-snapshot.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/snapshot-export.sh b/.github/scripts/snapshot-export.sh index 47989f9..7683844 100755 --- a/.github/scripts/snapshot-export.sh +++ b/.github/scripts/snapshot-export.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Export $DATADIR as /jochemnet/$CLIENT// with its metadata and head block # (benchmarkoor snapshot export). The head block is geth's, from the snapshot-tail artifact. -# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET +# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET, +# optional SNAPSHOT_PREFIX (a scratch prefix for a test run, e.g. "e2e/"). set -euo pipefail blk=$SNAPSHOT_TAIL_DIR/head.json meta=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json @@ -19,5 +20,5 @@ jq -n --arg img "$SNAPSHOT_IMAGE" --arg release "$RELEASE" --arg eest "$EEST" -- > "$RUNNER_TEMP/snapshot/metadata.json" sudo -E "$RUNNER_TEMP/benchmarkoor" snapshot export --client "$CLIENT" --datadir "$DATADIR" \ - --network jochemnet --block "$head" --bucket "$BUCKET" \ + --network jochemnet --block "$head" --bucket "$BUCKET" ${SNAPSHOT_PREFIX:+--prefix "$SNAPSHOT_PREFIX"} \ --head-block-file "$blk" --metadata-file "$RUNNER_TEMP/snapshot/metadata.json" diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 36ec350..192bccf 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -178,7 +178,7 @@ jobs: --datadir /data --override.amsterdam="$AMSTERDAM_TIME" --nodiscover --maxpeers 0 --port 0 \ --http --http.addr 127.0.0.1 --http.port 18545 --http.api eth --authrpc.port 18551 for _ in $(seq 120); do - curl -sf -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}' \ + curl -sf -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",true]}' \ http://127.0.0.1:18545 > "$W/head.json" && break sleep 5 done From 787b31165c0bc9eb53ed1dd94daa31b224da88cb Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 17:44:48 +0200 Subject: [PATCH 05/14] benchmarkoor-snapshot: check each datadir's layout before a client boots on it Found by the e2e prototype: geth on a flat instance dir opens the database but misses triedb/merkle.journal, then rewinds the head and truncates state history. --- .github/scripts/check-datadir-layout.sh | 18 ++++++++++++++++++ .github/workflows/benchmarkoor-snapshot.yaml | 6 ++++++ 2 files changed, 24 insertions(+) create mode 100755 .github/scripts/check-datadir-layout.sh diff --git a/.github/scripts/check-datadir-layout.sh b/.github/scripts/check-datadir-layout.sh new file mode 100755 index 0000000..222b95c --- /dev/null +++ b/.github/scripts/check-datadir-layout.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Fail before a client boots on a datadir laid out differently from what it is started with. +# geth started with --datadir on a flat instance dir (chaindata/ at the top) still opens the +# database, but looks for triedb/merkle.journal under /geth/, discards the state it +# holds, rewinds the head and truncates state history: the datadir is spoiled, not just unused. +# check-datadir-layout.sh +set -euo pipefail +c=${1:?client} d=${2:?datadir} +need() { [ -e "$d/$1" ] || { echo "::error::$c datadir $d has no $1 (wrong layout)"; exit 1; }; } +case $c in + geth) need geth/chaindata + [ ! -e "$d/chaindata" ] || { echo "::error::$d/chaindata: a flat instance dir, geth wants it under $d/geth/"; exit 1; } ;; + nethermind) need nethermind_db/mainnet/state ;; + besu) need database ;; + reth|erigon|ethrex) echo "$c datadir layout not checked (unverified)"; exit 0 ;; + *) echo "::error::unknown client $c"; exit 1 ;; +esac +echo "$c datadir layout ok: $d" diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 192bccf..3a9bab4 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -139,6 +139,9 @@ jobs: { echo "W=$W"; echo "SNAPSHOT_BASE_BUNDLE=$W/release/pre_run_bundle"; echo "SNAPSHOT_TAIL_DIR=$W/tail"; } >> "$GITHUB_ENV" .github/scripts/release-pre-run-bundle.sh "${{ inputs.release }}" "$W/release" + - name: Datadir layout + run: .github/scripts/check-datadir-layout.sh "$CLIENT" "$DATADIR" + - name: Replay the release, ramp the gas limit down uses: ethpandaops/benchmarkoor@master with: @@ -251,6 +254,9 @@ jobs: name: snapshot-tail path: ${{ env.SNAPSHOT_TAIL_DIR }} + - name: Datadir layout + run: .github/scripts/check-datadir-layout.sh "$CLIENT" "$DATADIR" + - name: Replay the release and the ramp uses: ethpandaops/benchmarkoor@master with: From 92ff2d4838b9d0d775bc56ec5deb16e1403f319b Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 8 Oct 2026 20:38:44 +0200 Subject: [PATCH 06/14] benchmarkoor-snapshot: db compact runs the release image that wrote the datadir Without --image the CLI pulled ethereum/client-go:stable (1.17.8 today) for a datadir written by v1.17.7 (found by the e2e prototype); a newer geth can migrate the database format on open. --- .github/workflows/benchmarkoor-snapshot.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 3a9bab4..3c3105a 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -200,8 +200,10 @@ jobs: path: ${{ env.SNAPSHOT_TAIL_DIR }} if-no-files-found: error + # --image: the release that wrote the datadir compacts it; the CLI default is the client's + # :stable, a newer geth that may migrate the database format on open. - name: Compact - run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" - name: Export if: inputs.publish @@ -279,7 +281,7 @@ jobs: # ethrex has no compaction; reth (MDBX) is a logged no-op. - name: Compact if: matrix.client != 'ethrex' - run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --compaction-image "$COMPACTION_IMAGE" + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" --compaction-image "$COMPACTION_IMAGE" - name: Export if: inputs.publish From 06fdb0ded92b2f46cd16a5715150434406c20d42 Mon Sep 17 00:00:00 2001 From: Stefan Date: Sun, 11 Oct 2026 12:01:31 +0200 Subject: [PATCH 07/14] benchmarkoor-snapshot: geth-replay target for a rebuild from an existing tail; compaction notes per client reth compacts (benchmarkoor #351: reth db copy --compact), ethrex does not: its TRANSACTION_LOCATIONS family has a merge operator ldb lacks, so the generic RocksDB compaction refuses the database. --- .github/workflows/benchmarkoor-snapshot.yaml | 4 +++- configs/snapshots/jochemnet/v1/builder.yaml | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 3c3105a..101c450 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -278,7 +278,9 @@ jobs: --skip-eest-payload-build --limit-pre-runs-target=${{ matrix.client }}-replay - # ethrex has no compaction; reth (MDBX) is a logged no-op. + # reth: `reth db copy --compact` (mdbx_copy, copy-and-swap); erigon: `erigon db compact`; + # nethermind/besu: ldb per column family. ethrex is RocksDB with a merge operator ldb + # lacks (tx_locations_merge), so the generic compaction refuses it: exported as replayed. - name: Compact if: matrix.client != 'ethrex' run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" --compaction-image "$COMPACTION_IMAGE" diff --git a/configs/snapshots/jochemnet/v1/builder.yaml b/configs/snapshots/jochemnet/v1/builder.yaml index 46c910b..ff43104 100644 --- a/configs/snapshots/jochemnet/v1/builder.yaml +++ b/configs/snapshots/jochemnet/v1/builder.yaml @@ -35,6 +35,22 @@ builder: - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} - --cache.gc=0 + # A geth rebuild from an existing tail (a re-run that must land on the same + # head as the other clients): replays the release and the tail like the + # others, with the ramp's --cache.gc=0 so the journal stays at 128 layers. + - name: geth-replay + filler_client: geth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${GETH_SNAPSHOT_DIR} + genesis: ${GETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + - --cache.gc=0 + # Every other client replays the release, then the ramp, onto its own # jochemnet snapshot. # Each payload must come back VALID, so each client re-derives geth's state From 747c0d34deb462a2e8a7f64a118fb3a0811ecb1c Mon Sep 17 00:00:00 2001 From: Stefan Date: Sun, 11 Oct 2026 12:29:03 +0200 Subject: [PATCH 08/14] benchmarkoor-snapshot: check the reth, erigon and ethrex datadir layouts too From jochemnet's archives, extracted on the build droplets: reth v2 has static_files/ and rocksdb/ at the root, erigon's pruned archive is the datadir (chaindata/, snapshots/), ethrex's is the RocksDB directory that lives at /chain-1. --- .github/scripts/check-datadir-layout.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/check-datadir-layout.sh b/.github/scripts/check-datadir-layout.sh index 222b95c..917b923 100755 --- a/.github/scripts/check-datadir-layout.sh +++ b/.github/scripts/check-datadir-layout.sh @@ -12,7 +12,12 @@ case $c in [ ! -e "$d/chaindata" ] || { echo "::error::$d/chaindata: a flat instance dir, geth wants it under $d/geth/"; exit 1; } ;; nethermind) need nethermind_db/mainnet/state ;; besu) need database ;; - reth|erigon|ethrex) echo "$c datadir layout not checked (unverified)"; exit 0 ;; + # jochemnet's reth v2 archive: static_files/, rocksdb/ and the MDBX db at the datadir root. + reth) need static_files; need rocksdb ;; + # erigon's pruned archive is its datadir: chaindata/ (mdbx) and snapshots/ (segments). + erigon) need chaindata; need snapshots ;; + # ethrex keeps its RocksDB at /chain-1; the archive is that directory's contents. + ethrex) need chain-1/CURRENT ;; *) echo "::error::unknown client $c"; exit 1 ;; esac echo "$c datadir layout ok: $d" From eaa5cff2c5e0c4ec963d48b01bd381c826ae52dc Mon Sep 17 00:00:00 2001 From: Stefan Date: Sun, 11 Oct 2026 22:00:27 +0200 Subject: [PATCH 09/14] snapshot: persist Nethermind state and gate export on offline reopen --- .github/scripts/snapshot-reopen-nethermind.sh | 70 +++++++++++++++++++ .github/workflows/benchmarkoor-snapshot.yaml | 13 ++++ configs/snapshots/jochemnet/v1/builder.yaml | 9 ++- 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/snapshot-reopen-nethermind.sh diff --git a/.github/scripts/snapshot-reopen-nethermind.sh b/.github/scripts/snapshot-reopen-nethermind.sh new file mode 100755 index 0000000..f237e3a --- /dev/null +++ b/.github/scripts/snapshot-reopen-nethermind.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Reopen the stopped snapshot with the consumer chainspec before publication. +# Env: DATADIR, SNAPSHOT_IMAGE, SNAPSHOT_TAIL_DIR, AMSTERDAM_TIME, W. +set -euo pipefail +DATADIR=${DATADIR:?datadir}; SNAPSHOT_IMAGE=${SNAPSHOT_IMAGE:?image} +GENESIS=${W:?work directory}/nethermind-reopen-genesis.json +OUT=$W/nethermind-reopen +expected=${SNAPSHOT_TAIL_DIR:?tail directory}/head.json +name=snapshot-nethermind-reopen-${GITHUB_RUN_ID:-$$} +url=$(sed -n 's/^ *GETH_GENESIS: *//p' configs/datadirs/jochemnet/v1/global.yaml) +[ -n "$url" ] +curl -fsSL --max-time 60 "$url" | jq --argjson ams "${AMSTERDAM_TIME:?fork timestamp}" '.config.amsterdamTime = $ams' > "$GENESIS" +mkdir -p "$OUT" +rm -f "$OUT/PASS" +want=$(jq -er .result.hash "$expected") +number=$(jq -er .result.number "$expected") +state_root=$(jq -er .result.stateRoot "$expected") +DBREL=nethermind_db/mainnet +[ ! -d "$DATADIR/mainnet" ] || DBREL=mainnet +[ -d "$DATADIR/$DBREL" ] +[ "$(jq -r .config.amsterdamTime "$GENESIS")" = "$AMSTERDAM_TIME" ] +[ -z "$(docker ps -aq --filter "name=^${name}$")" ] +cleanup() { + docker stop -t 600 "$name" > "$OUT/stop.log" 2>&1 || true + docker logs "$name" > "$OUT/container.log" 2>&1 || true + docker inspect -f '{{.State.ExitCode}}' "$name" > "$OUT/exit-code" 2>/dev/null || true + docker rm "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT +docker run -d --name "$name" --user 0:0 --network host \ + -v "$DATADIR:/data" -v "$GENESIS:/network-genesis.json:ro" "$SNAPSHOT_IMAGE" \ + --config=mainnet --datadir=/data --Init.ChainSpecPath=/network-genesis.json --Merge.TerminalTotalDifficulty=0 \ + --Init.BaseDbPath="/data/$DBREL" --FlatDb.Enabled=true \ + --Network.MaxActivePeers=0 --Init.DiscoveryEnabled=false \ + --Network.DiscoveryPort=30399 --Network.P2PPort=30399 \ + --JsonRpc.Enabled=true --JsonRpc.Host=127.0.0.1 --JsonRpc.Port=18545 \ + --JsonRpc.EngineHost=127.0.0.1 --JsonRpc.EnginePort=18551 \ + --Sync.NetworkingEnabled=false --Sync.FastSync=false --Sync.SnapSync=false +for ((n=0; n<180; n++)); do + if curl -fsS --max-time 5 -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}' \ + http://127.0.0.1:18545 > "$OUT/head.tmp.json" 2>/dev/null; then + mv "$OUT/head.tmp.json" "$OUT/head.json" + jq -c '.result | {number,hash,stateRoot}' "$OUT/head.json" + if jq -e --arg want "$want" --arg number "$number" --arg root "$state_root" '.result.hash == $want and .result.number == $number and .result.stateRoot == $root' "$OUT/head.json" >/dev/null; then + curl -fsS --max-time 10 -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_config","params":[]}' \ + http://127.0.0.1:18545 > "$OUT/eth-config.json" + activation=$(jq -er '.result.current.activationTime' "$OUT/eth-config.json") + [[ "$activation" =~ ^(0x[0-9a-fA-F]+|[0-9]+)$ ]] + [ "$((activation))" -eq "$AMSTERDAM_TIME" ] + for addr in 0x0000bff46984e3725691fa540a8c7589300d8282 0x000064d678505ad48f8ccb093bc65613800e8282; do + curl -fsS --max-time 10 -H 'Content-Type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getCode\",\"params\":[\"$addr\",\"latest\"]}" \ + http://127.0.0.1:18545 > "$OUT/code-$addr.json" + jq -e '.result | type == "string" and length > 2' "$OUT/code-$addr.json" >/dev/null + done + cleanup + trap - EXIT + case "$(cat "$OUT/exit-code")" in 0|130|143) ;; *) exit 1 ;; esac + date -u +%FT%TZ > "$OUT/PASS" + echo 'PASS: Nethermind reopens offline at the snapshot head' + exit 0 + fi + fi + [ "$(docker inspect -f '{{.State.Running}}' "$name")" = true ] || break + sleep 5 +done +echo 'FAIL: Nethermind did not reopen at the required head' >&2 +exit 1 diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 101c450..4644a9e 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -285,6 +285,10 @@ jobs: if: matrix.client != 'ethrex' run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" --compaction-image "$COMPACTION_IMAGE" + - name: Reopen Nethermind after compaction + if: matrix.client == 'nethermind' + run: .github/scripts/snapshot-reopen-nethermind.sh + - name: Export if: inputs.publish env: @@ -297,9 +301,18 @@ jobs: BUCKET: ${{ inputs.bucket }} run: .github/scripts/snapshot-export.sh + - name: Nethermind reopen evidence + if: always() && matrix.client == 'nethermind' + uses: actions/upload-artifact@v4 + with: + name: snapshot-nethermind-reopen + path: ${{ env.W }}/nethermind-reopen + if-no-files-found: ignore + - name: Clean up if: always() run: | + docker rm -f "snapshot-nethermind-reopen-${GITHUB_RUN_ID}" >/dev/null 2>&1 || true sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y sudo rm -rf "$RUNNER_TEMP/snapshot" diff --git a/configs/snapshots/jochemnet/v1/builder.yaml b/configs/snapshots/jochemnet/v1/builder.yaml index ff43104..3ea7a98 100644 --- a/configs/snapshots/jochemnet/v1/builder.yaml +++ b/configs/snapshots/jochemnet/v1/builder.yaml @@ -70,8 +70,13 @@ builder: - --Init.BaseDbPath=/data - --FlatDb.Enabled=true - --FlatDb.CompactionOffset=0 - - --FlatDb.MinReorgDepth=1024 - - --FlatDb.MaxReorgDepth=1024 + # A distributable snapshot must reopen without replaying a 1024-block + # flat-state window. Persist finalized state during the snapshot build; + # these settings do not change the benchmark or consumer defaults. + - --FlatDb.CompactSize=1 + - --FlatDb.MinReorgDepth=0 + - --FlatDb.MaxReorgDepth=0 + - --FlatDb.EnableLongFinality=false - name: besu-replay filler_client: besu From a0d9526777580768360572f89c6f4a6d22b44dc8 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 00:48:57 +0200 Subject: [PATCH 10/14] Normalize Geth alloc addresses for Nethermind reopen gate --- .github/scripts/snapshot-reopen-nethermind.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/snapshot-reopen-nethermind.sh b/.github/scripts/snapshot-reopen-nethermind.sh index f237e3a..c7c1594 100755 --- a/.github/scripts/snapshot-reopen-nethermind.sh +++ b/.github/scripts/snapshot-reopen-nethermind.sh @@ -9,7 +9,11 @@ expected=${SNAPSHOT_TAIL_DIR:?tail directory}/head.json name=snapshot-nethermind-reopen-${GITHUB_RUN_ID:-$$} url=$(sed -n 's/^ *GETH_GENESIS: *//p' configs/datadirs/jochemnet/v1/global.yaml) [ -n "$url" ] -curl -fsSL --max-time 60 "$url" | jq --argjson ams "${AMSTERDAM_TIME:?fork timestamp}" '.config.amsterdamTime = $ams' > "$GENESIS" +# Geth permits unprefixed alloc addresses; Nethermind's genesis reader requires 0x. +curl -fsSL --max-time 60 "$url" | jq --argjson ams "${AMSTERDAM_TIME:?fork timestamp}" ' + .config.amsterdamTime = $ams | + .alloc |= with_entries(.key |= if startswith("0x") then . else "0x" + . end) +' > "$GENESIS" mkdir -p "$OUT" rm -f "$OUT/PASS" want=$(jq -er .result.hash "$expected") From 9b44aea410556a26b0c82aa3f77e42b758ffd907 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 10:17:07 +0200 Subject: [PATCH 11/14] Use the ldb compaction image only for Nethermind and Besu --- .github/workflows/benchmarkoor-snapshot.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index 4644a9e..ac962e6 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -283,7 +283,12 @@ jobs: # lacks (tx_locations_merge), so the generic compaction refuses it: exported as replayed. - name: Compact if: matrix.client != 'ethrex' - run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" --compaction-image "$COMPACTION_IMAGE" + run: | + extra=() + case "$CLIENT" in + nethermind|besu) extra+=(--compaction-image "$COMPACTION_IMAGE") ;; + esac + sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" "${extra[@]}" - name: Reopen Nethermind after compaction if: matrix.client == 'nethermind' From 87dbda12e250a46b5361cf89ac0881f5dd74777d Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 10:35:58 +0200 Subject: [PATCH 12/14] snapshot: remove unused prefix from production export helper --- .github/scripts/snapshot-export.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/snapshot-export.sh b/.github/scripts/snapshot-export.sh index 7683844..3b23fb5 100755 --- a/.github/scripts/snapshot-export.sh +++ b/.github/scripts/snapshot-export.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Export $DATADIR as /jochemnet/$CLIENT// with its metadata and head block # (benchmarkoor snapshot export). The head block is geth's, from the snapshot-tail artifact. -# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET, -# optional SNAPSHOT_PREFIX (a scratch prefix for a test run, e.g. "e2e/"). +# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET. +# Production path matches the workflow latest job; scratch exports use the CLI directly. set -euo pipefail blk=$SNAPSHOT_TAIL_DIR/head.json meta=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json @@ -20,5 +20,5 @@ jq -n --arg img "$SNAPSHOT_IMAGE" --arg release "$RELEASE" --arg eest "$EEST" -- > "$RUNNER_TEMP/snapshot/metadata.json" sudo -E "$RUNNER_TEMP/benchmarkoor" snapshot export --client "$CLIENT" --datadir "$DATADIR" \ - --network jochemnet --block "$head" --bucket "$BUCKET" ${SNAPSHOT_PREFIX:+--prefix "$SNAPSHOT_PREFIX"} \ + --network jochemnet --block "$head" --bucket "$BUCKET" \ --head-block-file "$blk" --metadata-file "$RUNNER_TEMP/snapshot/metadata.json" From 28ac19d4d3ce523babd5d75c8c05baef69e0863e Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 11:28:26 +0200 Subject: [PATCH 13/14] fix(snapshot): compact only through integrated client commands --- .github/workflows/benchmarkoor-snapshot.yaml | 30 ++++++++------------ README.md | 9 +++--- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index ac962e6..a08876f 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -3,12 +3,12 @@ name: benchmarkoor-snapshot # Turns a jochemnet benchmarkoor-release into a shadowfork snapshot every client # can boot without a pre-run of its own (msf-2): the release's pre-run applied to # each client's jochemnet datadir, the gas limit walked down, the datadir -# compacted, and the result published to /jochemnet///. +# compacted with client commands where supported, and the result published to /jochemnet///. # # prepare release manifest → base and head blocks, context; bucket write probe # geth replay the release's pre-run, ramp the gas limit down (the tail # bundle), reopen at the head, compact, export -# replay every other client: release pre-run + tail, compact, export +# replay every other client: release pre-run + tail, compact if supported, export # latest only once every client is up: /jochemnet//latest # # Each payload must come back VALID on every client, so every image has geth's @@ -36,9 +36,6 @@ on: images: description: "Client images (JSON): the releases the devnets run, which open and publish each datadir" default: '{"geth": "ethereum/client-go:v1.17.7", "nethermind": "nethermind/nethermind:2.1.0", "besu": "hyperledger/besu:26.9.0", "reth": "ghcr.io/paradigmxyz/reth:v2.7.0", "erigon": "erigontech/erigon:v3.7.1", "ethrex": "ghcr.io/lambdaclass/ethrex:29.0.1"}' - compaction-image: - description: "RocksDB ldb image for nethermind/besu compaction" - default: "ghcr.io/ethpandaops/benchmarkoor-rocksdb-ldb:11.8.1" bucket: description: "Bucket published to as jochemnet///" default: "ethpandaops-shadowfork-images" @@ -239,7 +236,6 @@ jobs: CLIENT: ${{ matrix.client }} SNAPSHOT_IMAGE: ${{ fromJSON(inputs.images)[matrix.client] }} SNAPSHOT_GAS_LIMIT: ${{ inputs.gas-limit }} - COMPACTION_IMAGE: ${{ inputs.compaction-image }} DATADIR: /schelk/snapshots/${{ matrix.client }}/jochemnet/${{ needs.prepare.outputs.base }} AMSTERDAM_TIME: ${{ needs.prepare.outputs.ams }} steps: @@ -278,19 +274,17 @@ jobs: --skip-eest-payload-build --limit-pre-runs-target=${{ matrix.client }}-replay - # reth: `reth db copy --compact` (mdbx_copy, copy-and-swap); erigon: `erigon db compact`; - # nethermind/besu: ldb per column family. ethrex is RocksDB with a merge operator ldb - # lacks (tx_locations_merge), so the generic compaction refuses it: exported as replayed. - - name: Compact - if: matrix.client != 'ethrex' - run: | - extra=() - case "$CLIENT" in - nethermind|besu) extra+=(--compaction-image "$COMPACTION_IMAGE") ;; - esac - sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" "${extra[@]}" + # Only run compaction provided by the client. Other datadirs are exported + # after replay without external database tools or manual file replacement. + - name: Compact with client command + if: matrix.client == 'erigon' + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" + + - name: Report skipped compaction + if: matrix.client != 'erigon' + run: echo "Compaction skipped for $CLIENT; no client-owned compaction command is integrated." - - name: Reopen Nethermind after compaction + - name: Reopen Nethermind before export if: matrix.client == 'nethermind' run: .github/scripts/snapshot-reopen-nethermind.sh diff --git a/README.md b/README.md index 6a0071f..cb7b2bb 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ All workflows are `workflow_dispatch`-only and must be dispatched from the defau | `benchmarkoor-build-state-actor.yaml` | Build **only** the state-actor datadir (no payload fill). Has a `force` input to rebuild over an already-populated/partial datadir. | | `benchmarkoor-release.yaml` | Promote the filled payloads from a build run into a GitHub Release. | | `benchmarkoor-run.yaml` | Build the datadir, replay the payloads, and upload results to S3. | -| `benchmarkoor-snapshot.yaml` | Turn a jochemnet release into a shadowfork snapshot: every client's jochemnet datadir advanced by the release's pre-run, its gas limit walked down, compacted, and published as `/jochemnet///`. See below. | +| `benchmarkoor-snapshot.yaml` | Turn a jochemnet release into a shadowfork snapshot: every client's jochemnet datadir advanced by the release's pre-run, its gas limit walked down, compacted where supported, and published as `/jochemnet///`. See below. | Common inputs: `clients` (JSON array), `snapshot`, `context`, `subdir`, `test-type`, and `instance-id`. `benchmarkoor-build-state-actor.yaml` takes only `snapshot` + `clients` (plus `force`), since the test context is irrelevant to a datadir-only build. @@ -106,9 +106,10 @@ release's contracts without a pre-run of its own. Given a release tag: geth must come back at the tail's head; its head block is what every client publishes. 2. **every other client** replays the release's pre-run and then the tail onto its own snapshot. Every payload must be VALID, so every image agrees with geth at every block. -3. Each datadir is **compacted** (`benchmarkoor db compact`; ldb per column family for - nethermind/besu, none for ethrex) so the release's contracts sit in the same DB tiers as old - state and read no faster than mainnet state would, then **exported** (`benchmarkoor snapshot export`). +3. **Geth and Erigon** are compacted with their own commands (`benchmarkoor db compact`). + Compaction is skipped for the other clients; no external database tool or manual file swap + is used. Nethermind must reopen offline at the expected head before export. Each datadir + is then **exported** (`benchmarkoor snapshot export`). 4. `latest` is written last, once every client's archive is up. Runs on the jochemnet hosts, whose schelk baseline is the release's base snapshot (checked against From d7da397de6f7170ae5ada0cca7cdf0143128eba9 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 13:04:22 +0200 Subject: [PATCH 14/14] fix(snapshot): restore CLI after benchmarkoor action cleanup --- .github/scripts/snapshot-extract-benchmarkoor.sh | 14 ++++++++++++++ .github/workflows/benchmarkoor-snapshot.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100755 .github/scripts/snapshot-extract-benchmarkoor.sh diff --git a/.github/scripts/snapshot-extract-benchmarkoor.sh b/.github/scripts/snapshot-extract-benchmarkoor.sh new file mode 100755 index 0000000..e1678f6 --- /dev/null +++ b/.github/scripts/snapshot-extract-benchmarkoor.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# The benchmarkoor action deletes its binary. Restore it from the same local +# image for the subsequent compaction/export steps; do not pull a moving tag again. +# Env: RUNNER_TEMP, BENCHMARKOOR_IMAGE (empty selects the action's local build). +set -euo pipefail +image=${BENCHMARKOOR_IMAGE:-benchmarkoor:local} +image_id=$(docker image inspect --format '{{.Id}}' "$image") +cid=$(docker create "$image_id") +trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT +bin=${RUNNER_TEMP:?runner temp}/benchmarkoor +docker cp "$cid:/usr/local/bin/benchmarkoor" "$bin" +chmod +x "$bin" +"$bin" db compact --help >/dev/null +"$bin" snapshot export --help >/dev/null diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml index a08876f..b262613 100644 --- a/.github/workflows/benchmarkoor-snapshot.yaml +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -158,6 +158,12 @@ jobs: --skip-eest-payload-build --limit-pre-runs-target=geth-ramp + # The composite action removes its binary during cleanup. + - name: Restore benchmarkoor CLI + env: + BENCHMARKOOR_IMAGE: ${{ inputs.image }} + run: .github/scripts/snapshot-extract-benchmarkoor.sh + - name: Check the tail env: RELEASE_HEAD: ${{ needs.prepare.outputs.head }} @@ -220,6 +226,7 @@ jobs: docker rm -f snapshot-geth-reopen >/dev/null 2>&1 || true sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y sudo rm -rf "$RUNNER_TEMP/snapshot" + rm -f "$RUNNER_TEMP/benchmarkoor" replay: needs: [prepare, geth] @@ -274,6 +281,12 @@ jobs: --skip-eest-payload-build --limit-pre-runs-target=${{ matrix.client }}-replay + # The composite action removes its binary during cleanup. + - name: Restore benchmarkoor CLI + env: + BENCHMARKOOR_IMAGE: ${{ inputs.image }} + run: .github/scripts/snapshot-extract-benchmarkoor.sh + # Only run compaction provided by the client. Other datadirs are exported # after replay without external database tools or manual file replacement. - name: Compact with client command @@ -314,6 +327,7 @@ jobs: docker rm -f "snapshot-nethermind-reopen-${GITHUB_RUN_ID}" >/dev/null 2>&1 || true sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y sudo rm -rf "$RUNNER_TEMP/snapshot" + rm -f "$RUNNER_TEMP/benchmarkoor" # latest is written last and only once every client is up, so a consumer of it # never meets a height some client lacks.