diff --git a/.github/scripts/check-datadir-layout.sh b/.github/scripts/check-datadir-layout.sh new file mode 100755 index 0000000..917b923 --- /dev/null +++ b/.github/scripts/check-datadir-layout.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Fail before a client boots on a datadir laid out differently from what it is started with. +# geth started with --datadir on a flat instance dir (chaindata/ at the top) still opens the +# database, but looks for triedb/merkle.journal under /geth/, discards the state it +# holds, rewinds the head and truncates state history: the datadir is spoiled, not just unused. +# check-datadir-layout.sh +set -euo pipefail +c=${1:?client} d=${2:?datadir} +need() { [ -e "$d/$1" ] || { echo "::error::$c datadir $d has no $1 (wrong layout)"; exit 1; }; } +case $c in + geth) need geth/chaindata + [ ! -e "$d/chaindata" ] || { echo "::error::$d/chaindata: a flat instance dir, geth wants it under $d/geth/"; exit 1; } ;; + nethermind) need nethermind_db/mainnet/state ;; + besu) need database ;; + # jochemnet's reth v2 archive: static_files/, rocksdb/ and the MDBX db at the datadir root. + reth) need static_files; need rocksdb ;; + # erigon's pruned archive is its datadir: chaindata/ (mdbx) and snapshots/ (segments). + erigon) need chaindata; need snapshots ;; + # ethrex keeps its RocksDB at /chain-1; the archive is that directory's contents. + ethrex) need chain-1/CURRENT ;; + *) echo "::error::unknown client $c"; exit 1 ;; +esac +echo "$c datadir layout ok: $d" diff --git a/.github/scripts/release-pre-run-bundle.sh b/.github/scripts/release-pre-run-bundle.sh new file mode 100755 index 0000000..875639a --- /dev/null +++ b/.github/scripts/release-pre-run-bundle.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Fetch a benchmarkoor-release's pre-run bundle into /pre_run_bundle, checked +# against the release's sha256. Prints the bundle's pre-run.meta.json. +# release-pre-run-bundle.sh +set -euo pipefail +tag=${1:?release tag} dest=${2:?dest dir} +base="https://github.com/${GITHUB_REPOSITORY:-ethpandaops/benchmarkoor-tests}/releases/download/$tag" +dl="$dest/download" +mkdir -p "$dl" + +curl -fsSL --retry 5 "$base/manifest.json" -o "$dl/manifest.json" +# Every client's run replays the geth pre-run (test-source.*.runner.yaml), so it is the only one. +asset=$(jq -ce '.assets[] | select(.client == "geth")' "$dl/manifest.json") +tarball=$(jq -r .tarball <<<"$asset") +mapfile -t files < <(jq -r '.files[]' <<<"$asset") + +for f in "${files[@]}"; do curl -fsSL --retry 5 "$base/$f" -o "$dl/$f"; done +# Assets over 2 GB are split into .part-NNN; an unsplit asset is the tarball itself. +[ "${#files[@]}" -eq 1 ] && [ "${files[0]}" = "$tarball" ] || { (cd "$dl" && cat "${files[@]}") > "$dl/$tarball"; } +echo "$(jq -r .sha256 <<<"$asset") $dl/$tarball" | sha256sum -c --quiet - + +tar -xzf "$dl/$tarball" -C "$dest" --strip-components=3 benchmarkoor-build-artifacts/pre-runs/geth/pre_run_bundle +rm -rf "$dl" +cat "$dest/pre_run_bundle/pre-run.meta.json" diff --git a/.github/scripts/snapshot-export.sh b/.github/scripts/snapshot-export.sh new file mode 100755 index 0000000..3b23fb5 --- /dev/null +++ b/.github/scripts/snapshot-export.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Export $DATADIR as /jochemnet/$CLIENT// with its metadata and head block +# (benchmarkoor snapshot export). The head block is geth's, from the snapshot-tail artifact. +# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET. +# Production path matches the workflow latest job; scratch exports use the CLI directly. +set -euo pipefail +blk=$SNAPSHOT_TAIL_DIR/head.json +meta=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json +head=$(( $(jq -r .result.number "$blk") )) + +jq -n --arg img "$SNAPSHOT_IMAGE" --arg release "$RELEASE" --arg eest "$EEST" --argjson base "$BASE" \ + --argjson head "$head" --arg hash "$(jq -r .result.hash "$blk")" --argjson ams "$AMSTERDAM_TIME" \ + --argjson gas "$SNAPSHOT_GAS_LIMIT" --argjson ramp "$(jq .payloads "$meta")" \ + --arg run "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" '{ + docker_image: $img, + shadowfork: {network: "jochemnet", base: $base, release: $release, eest: $eest, + head: $head, head_hash: $hash, gas_limit: $gas, ramp_blocks: $ramp, amsterdam_time: $ams, + filler: "geth", built_by: $run, + notes: "already on Amsterdam: a devnet must start on Gloas with amsterdamTime = amsterdam_time"}}' \ + > "$RUNNER_TEMP/snapshot/metadata.json" + +sudo -E "$RUNNER_TEMP/benchmarkoor" snapshot export --client "$CLIENT" --datadir "$DATADIR" \ + --network jochemnet --block "$head" --bucket "$BUCKET" \ + --head-block-file "$blk" --metadata-file "$RUNNER_TEMP/snapshot/metadata.json" diff --git a/.github/scripts/snapshot-extract-benchmarkoor.sh b/.github/scripts/snapshot-extract-benchmarkoor.sh new file mode 100755 index 0000000..e1678f6 --- /dev/null +++ b/.github/scripts/snapshot-extract-benchmarkoor.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# The benchmarkoor action deletes its binary. Restore it from the same local +# image for the subsequent compaction/export steps; do not pull a moving tag again. +# Env: RUNNER_TEMP, BENCHMARKOOR_IMAGE (empty selects the action's local build). +set -euo pipefail +image=${BENCHMARKOOR_IMAGE:-benchmarkoor:local} +image_id=$(docker image inspect --format '{{.Id}}' "$image") +cid=$(docker create "$image_id") +trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT +bin=${RUNNER_TEMP:?runner temp}/benchmarkoor +docker cp "$cid:/usr/local/bin/benchmarkoor" "$bin" +chmod +x "$bin" +"$bin" db compact --help >/dev/null +"$bin" snapshot export --help >/dev/null diff --git a/.github/scripts/snapshot-reopen-nethermind.sh b/.github/scripts/snapshot-reopen-nethermind.sh new file mode 100755 index 0000000..c7c1594 --- /dev/null +++ b/.github/scripts/snapshot-reopen-nethermind.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Reopen the stopped snapshot with the consumer chainspec before publication. +# Env: DATADIR, SNAPSHOT_IMAGE, SNAPSHOT_TAIL_DIR, AMSTERDAM_TIME, W. +set -euo pipefail +DATADIR=${DATADIR:?datadir}; SNAPSHOT_IMAGE=${SNAPSHOT_IMAGE:?image} +GENESIS=${W:?work directory}/nethermind-reopen-genesis.json +OUT=$W/nethermind-reopen +expected=${SNAPSHOT_TAIL_DIR:?tail directory}/head.json +name=snapshot-nethermind-reopen-${GITHUB_RUN_ID:-$$} +url=$(sed -n 's/^ *GETH_GENESIS: *//p' configs/datadirs/jochemnet/v1/global.yaml) +[ -n "$url" ] +# Geth permits unprefixed alloc addresses; Nethermind's genesis reader requires 0x. +curl -fsSL --max-time 60 "$url" | jq --argjson ams "${AMSTERDAM_TIME:?fork timestamp}" ' + .config.amsterdamTime = $ams | + .alloc |= with_entries(.key |= if startswith("0x") then . else "0x" + . end) +' > "$GENESIS" +mkdir -p "$OUT" +rm -f "$OUT/PASS" +want=$(jq -er .result.hash "$expected") +number=$(jq -er .result.number "$expected") +state_root=$(jq -er .result.stateRoot "$expected") +DBREL=nethermind_db/mainnet +[ ! -d "$DATADIR/mainnet" ] || DBREL=mainnet +[ -d "$DATADIR/$DBREL" ] +[ "$(jq -r .config.amsterdamTime "$GENESIS")" = "$AMSTERDAM_TIME" ] +[ -z "$(docker ps -aq --filter "name=^${name}$")" ] +cleanup() { + docker stop -t 600 "$name" > "$OUT/stop.log" 2>&1 || true + docker logs "$name" > "$OUT/container.log" 2>&1 || true + docker inspect -f '{{.State.ExitCode}}' "$name" > "$OUT/exit-code" 2>/dev/null || true + docker rm "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT +docker run -d --name "$name" --user 0:0 --network host \ + -v "$DATADIR:/data" -v "$GENESIS:/network-genesis.json:ro" "$SNAPSHOT_IMAGE" \ + --config=mainnet --datadir=/data --Init.ChainSpecPath=/network-genesis.json --Merge.TerminalTotalDifficulty=0 \ + --Init.BaseDbPath="/data/$DBREL" --FlatDb.Enabled=true \ + --Network.MaxActivePeers=0 --Init.DiscoveryEnabled=false \ + --Network.DiscoveryPort=30399 --Network.P2PPort=30399 \ + --JsonRpc.Enabled=true --JsonRpc.Host=127.0.0.1 --JsonRpc.Port=18545 \ + --JsonRpc.EngineHost=127.0.0.1 --JsonRpc.EnginePort=18551 \ + --Sync.NetworkingEnabled=false --Sync.FastSync=false --Sync.SnapSync=false +for ((n=0; n<180; n++)); do + if curl -fsS --max-time 5 -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}' \ + http://127.0.0.1:18545 > "$OUT/head.tmp.json" 2>/dev/null; then + mv "$OUT/head.tmp.json" "$OUT/head.json" + jq -c '.result | {number,hash,stateRoot}' "$OUT/head.json" + if jq -e --arg want "$want" --arg number "$number" --arg root "$state_root" '.result.hash == $want and .result.number == $number and .result.stateRoot == $root' "$OUT/head.json" >/dev/null; then + curl -fsS --max-time 10 -H 'Content-Type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_config","params":[]}' \ + http://127.0.0.1:18545 > "$OUT/eth-config.json" + activation=$(jq -er '.result.current.activationTime' "$OUT/eth-config.json") + [[ "$activation" =~ ^(0x[0-9a-fA-F]+|[0-9]+)$ ]] + [ "$((activation))" -eq "$AMSTERDAM_TIME" ] + for addr in 0x0000bff46984e3725691fa540a8c7589300d8282 0x000064d678505ad48f8ccb093bc65613800e8282; do + curl -fsS --max-time 10 -H 'Content-Type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getCode\",\"params\":[\"$addr\",\"latest\"]}" \ + http://127.0.0.1:18545 > "$OUT/code-$addr.json" + jq -e '.result | type == "string" and length > 2' "$OUT/code-$addr.json" >/dev/null + done + cleanup + trap - EXIT + case "$(cat "$OUT/exit-code")" in 0|130|143) ;; *) exit 1 ;; esac + date -u +%FT%TZ > "$OUT/PASS" + echo 'PASS: Nethermind reopens offline at the snapshot head' + exit 0 + fi + fi + [ "$(docker inspect -f '{{.State.Running}}' "$name")" = true ] || break + sleep 5 +done +echo 'FAIL: Nethermind did not reopen at the required head' >&2 +exit 1 diff --git a/.github/workflows/benchmarkoor-snapshot.yaml b/.github/workflows/benchmarkoor-snapshot.yaml new file mode 100644 index 0000000..b262613 --- /dev/null +++ b/.github/workflows/benchmarkoor-snapshot.yaml @@ -0,0 +1,358 @@ +name: benchmarkoor-snapshot + +# Turns a jochemnet benchmarkoor-release into a shadowfork snapshot every client +# can boot without a pre-run of its own (msf-2): the release's pre-run applied to +# each client's jochemnet datadir, the gas limit walked down, the datadir +# compacted with client commands where supported, and the result published to /jochemnet///. +# +# prepare release manifest → base and head blocks, context; bucket write probe +# geth replay the release's pre-run, ramp the gas limit down (the tail +# bundle), reopen at the head, compact, export +# replay every other client: release pre-run + tail, compact if supported, export +# latest only once every client is up: /jochemnet//latest +# +# Each payload must come back VALID on every client, so every image has geth's +# state root, receipts and block access list at every block. Runs on the +# jochemnet hosts, whose schelk baseline holds the release's base snapshot; the +# scratch is recovered afterwards so the next benchmark starts from the baseline. +# +# Needs benchmarkoor with pre_runs base_bundle and fill: false (#349), +# `db compact` (#351) and `snapshot export` (#350): until they merge, dispatch +# with image "" and a git-ref carrying all three. +run-name: "benchmarkoor-snapshot · ${{ inputs.release }} · gas=${{ inputs.gas-limit }} · publish=${{ inputs.publish }}" + +on: + workflow_dispatch: + inputs: + release: + description: "jochemnet benchmarkoor-release whose pre-run the snapshot carries" + default: "eest-payloads-jochemnet-v1-amsterdam-stateful-e269bb44-20260916-124530" + gas-limit: + description: "Gas limit of the published head (the release's payloads run at 1 TGas)" + default: "200000000" + clients: + description: "Clients replaying geth's chain (JSON array); geth always runs" + default: '["nethermind", "besu", "reth", "erigon", "ethrex"]' + images: + description: "Client images (JSON): the releases the devnets run, which open and publish each datadir" + default: '{"geth": "ethereum/client-go:v1.17.7", "nethermind": "nethermind/nethermind:2.1.0", "besu": "hyperledger/besu:26.9.0", "reth": "ghcr.io/paradigmxyz/reth:v2.7.0", "erigon": "erigontech/erigon:v3.7.1", "ethrex": "ghcr.io/lambdaclass/ethrex:29.0.1"}' + bucket: + description: "Bucket published to as jochemnet///" + default: "ethpandaops-shadowfork-images" + publish: + description: "Export to the bucket and write latest (false: build, check and compact only)" + type: boolean + default: true + image: + description: "Benchmarkoor docker image. Empty builds it from git-ref." + default: "ghcr.io/ethpandaops/benchmarkoor:master" + git-ref: + description: "Benchmarkoor git ref, used only when image is empty" + default: "" + timeout-minutes: + description: "Per client job" + default: "2880" + +concurrency: + group: benchmarkoor-snapshot + cancel-in-progress: false + +env: + CONFIG_BASE: https://raw.githubusercontent.com/ethpandaops/benchmarkoor-tests/${{ github.sha }}/configs + AWS_DEFAULT_REGION: auto + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + base: ${{ steps.release.outputs.base }} + head: ${{ steps.release.outputs.head }} + context: ${{ steps.release.outputs.context }} + subdir: ${{ steps.release.outputs.subdir }} + eest: ${{ steps.release.outputs.eest }} + ams: ${{ steps.release.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release + id: release + env: + TAG: ${{ inputs.release }} + run: | + m=$(curl -fsSL "https://github.com/${GITHUB_REPOSITORY}/releases/download/$TAG/manifest.json") + [ "$(jq -r .snapshot <<<"$m")" = jochemnet/v1 ] || { echo "::error::$TAG is not a jochemnet/v1 release"; exit 1; } + b=$(jq -ce '.assets[] | select(.client == "geth") | .build' <<<"$m") + base=$(jq -re .prerun_start_block <<<"$b") + # The hosts' schelk baseline is the snapshot named in the datadirs config; the + # release's pre-run has to start on it. + dir=$(sed -n 's/^ *GETH_SNAPSHOT_DIR: *//p' configs/datadirs/jochemnet/v1/global.yaml) + [ "${dir##*/}" = "$base" ] || { echo "::error::$TAG starts at $base, the hosts hold ${dir##*/}"; exit 1; } + # The release was built and is run on its context's schedule; so is the snapshot. + ctx="configs/contexts/$(jq -r .context <<<"$m")/$(jq -r .subdir <<<"$m")/global.yaml" + ams=$(sed -n 's/^ *AMSTERDAM_ACTIVATION_TS: *"\{0,1\}\([0-9]*\).*/\1/p' "$ctx") + [ -n "$ams" ] || { echo "::error::no AMSTERDAM_ACTIVATION_TS in $ctx"; exit 1; } + { + echo "ams=$ams" + echo "base=$base" + echo "head=$(jq -re .prerun_end_block <<<"$b")" + echo "context=$(jq -re .context <<<"$m")" + echo "subdir=$(jq -re .subdir <<<"$m")" + echo "eest=$(jq -re .eest_sha <<<"$m")" + } | tee -a "$GITHUB_OUTPUT" + + - name: Bucket write + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + BUCKET: ${{ inputs.bucket }} + run: | + key="jochemnet/.probe-${GITHUB_RUN_ID}" + if ! { echo probe | aws s3 cp - "s3://$BUCKET/$key" && aws s3 rm "s3://$BUCKET/$key"; }; then + echo "::error::the repo's S3 credentials cannot write $BUCKET"; exit 1 + fi + + geth: + needs: prepare + runs-on: + group: gasbenchmark + labels: + - benchmarkoor-v1-jochemnet-runner-geth + - benchmarkoor-v1-jochemnet-runner-benchmark-ci-3 + timeout-minutes: ${{ fromJSON(inputs.timeout-minutes) }} + env: + CLIENT: geth + SNAPSHOT_IMAGE: ${{ fromJSON(inputs.images).geth }} + SNAPSHOT_GAS_LIMIT: ${{ inputs.gas-limit }} + DATADIR: /schelk/snapshots/geth/jochemnet/${{ needs.prepare.outputs.base }} + AMSTERDAM_TIME: ${{ needs.prepare.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release pre-run + run: | + W=$RUNNER_TEMP/snapshot + { echo "W=$W"; echo "SNAPSHOT_BASE_BUNDLE=$W/release/pre_run_bundle"; echo "SNAPSHOT_TAIL_DIR=$W/tail"; } >> "$GITHUB_ENV" + .github/scripts/release-pre-run-bundle.sh "${{ inputs.release }}" "$W/release" + + - name: Datadir layout + run: .github/scripts/check-datadir-layout.sh "$CLIENT" "$DATADIR" + + - name: Replay the release, ramp the gas limit down + uses: ethpandaops/benchmarkoor@master + with: + mode: build + github-token: ${{ secrets.GITHUB_TOKEN }} + image: ${{ inputs.image }} + git-ref: ${{ inputs.git-ref }} + upload-artifacts: "false" + upload-build-artifacts: "false" + build-config-urls: >- + ${{ env.CONFIG_BASE }}/global.yaml, + ${{ env.CONFIG_BASE }}/datadirs/jochemnet/v1/global.yaml, + ${{ env.CONFIG_BASE }}/contexts/${{ needs.prepare.outputs.context }}/${{ needs.prepare.outputs.subdir }}/global.yaml, + ${{ env.CONFIG_BASE }}/snapshots/jochemnet/v1/builder.yaml + build-args: >- + --skip-state-actor-build + --skip-eest-payload-build + --limit-pre-runs-target=geth-ramp + + # The composite action removes its binary during cleanup. + - name: Restore benchmarkoor CLI + env: + BENCHMARKOOR_IMAGE: ${{ inputs.image }} + run: .github/scripts/snapshot-extract-benchmarkoor.sh + + - name: Check the tail + env: + RELEASE_HEAD: ${{ needs.prepare.outputs.head }} + run: | + m=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json + cat "$m" + [ "$(jq -r .start_block_number "$m")" = "$RELEASE_HEAD" ] || { echo "::error::the ramp does not start on the release head $RELEASE_HEAD"; exit 1; } + last=$(tail -n 2 "$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.request" | head -n 1 | jq -r '.params[0].gasLimit') + [ $((last)) -eq "$SNAPSHOT_GAS_LIMIT" ] || { echo "::error::head gas limit $((last)), want $SNAPSHOT_GAS_LIMIT"; exit 1; } + + # The head's block is what a snapshot consumer anchors on, and reading it from a + # restarted geth proves the datadir reopens at the head: the journal written on + # stop holds only the last 128 ramp blocks (--cache.gc=0 flushed the rest). + - name: Reopen at the head + run: | + want=$(jq -r .end_block_hash "$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json") + docker run -d --name snapshot-geth-reopen --network host -v "$DATADIR:/data" "$SNAPSHOT_IMAGE" \ + --datadir /data --override.amsterdam="$AMSTERDAM_TIME" --nodiscover --maxpeers 0 --port 0 \ + --http --http.addr 127.0.0.1 --http.port 18545 --http.api eth --authrpc.port 18551 + for _ in $(seq 120); do + curl -sf -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",true]}' \ + http://127.0.0.1:18545 > "$W/head.json" && break + sleep 5 + done + docker stop -t 600 snapshot-geth-reopen + code=$(docker inspect -f '{{.State.ExitCode}}' snapshot-geth-reopen) + docker logs --tail 30 snapshot-geth-reopen + docker rm snapshot-geth-reopen + [ "$code" = 0 ] || { echo "::error::geth exited $code"; exit 1; } + got=$(jq -r .result.hash "$W/head.json") + [ "$got" = "$want" ] || { echo "::error::geth reopened at $got, want $want"; exit 1; } + cp "$W/head.json" "$SNAPSHOT_TAIL_DIR/" + + - uses: actions/upload-artifact@v4 + with: + name: snapshot-tail + path: ${{ env.SNAPSHOT_TAIL_DIR }} + if-no-files-found: error + + # --image: the release that wrote the datadir compacts it; the CLI default is the client's + # :stable, a newer geth that may migrate the database format on open. + - name: Compact + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" + + - name: Export + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + RELEASE: ${{ inputs.release }} + EEST: ${{ needs.prepare.outputs.eest }} + BASE: ${{ needs.prepare.outputs.base }} + BUCKET: ${{ inputs.bucket }} + run: .github/scripts/snapshot-export.sh + + - name: Clean up + if: always() + run: | + docker rm -f snapshot-geth-reopen >/dev/null 2>&1 || true + sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y + sudo rm -rf "$RUNNER_TEMP/snapshot" + rm -f "$RUNNER_TEMP/benchmarkoor" + + replay: + needs: [prepare, geth] + strategy: + fail-fast: false + matrix: + client: ${{ fromJSON(inputs.clients) }} + runs-on: + group: gasbenchmark + labels: + - benchmarkoor-v1-jochemnet-runner-${{ matrix.client }} + timeout-minutes: ${{ fromJSON(inputs.timeout-minutes) }} + env: + CLIENT: ${{ matrix.client }} + SNAPSHOT_IMAGE: ${{ fromJSON(inputs.images)[matrix.client] }} + SNAPSHOT_GAS_LIMIT: ${{ inputs.gas-limit }} + DATADIR: /schelk/snapshots/${{ matrix.client }}/jochemnet/${{ needs.prepare.outputs.base }} + AMSTERDAM_TIME: ${{ needs.prepare.outputs.ams }} + steps: + - uses: actions/checkout@v5 + + - name: Release pre-run + run: | + W=$RUNNER_TEMP/snapshot + { echo "W=$W"; echo "SNAPSHOT_BASE_BUNDLE=$W/release/pre_run_bundle"; echo "SNAPSHOT_TAIL_DIR=$W/tail"; } >> "$GITHUB_ENV" + .github/scripts/release-pre-run-bundle.sh "${{ inputs.release }}" "$W/release" + + - uses: actions/download-artifact@v4 + with: + name: snapshot-tail + path: ${{ env.SNAPSHOT_TAIL_DIR }} + + - name: Datadir layout + run: .github/scripts/check-datadir-layout.sh "$CLIENT" "$DATADIR" + + - name: Replay the release and the ramp + uses: ethpandaops/benchmarkoor@master + with: + mode: build + github-token: ${{ secrets.GITHUB_TOKEN }} + image: ${{ inputs.image }} + git-ref: ${{ inputs.git-ref }} + upload-artifacts: "false" + upload-build-artifacts: "false" + build-config-urls: >- + ${{ env.CONFIG_BASE }}/global.yaml, + ${{ env.CONFIG_BASE }}/datadirs/jochemnet/v1/global.yaml, + ${{ env.CONFIG_BASE }}/contexts/${{ needs.prepare.outputs.context }}/${{ needs.prepare.outputs.subdir }}/global.yaml, + ${{ env.CONFIG_BASE }}/snapshots/jochemnet/v1/builder.yaml + build-args: >- + --skip-state-actor-build + --skip-eest-payload-build + --limit-pre-runs-target=${{ matrix.client }}-replay + + # The composite action removes its binary during cleanup. + - name: Restore benchmarkoor CLI + env: + BENCHMARKOOR_IMAGE: ${{ inputs.image }} + run: .github/scripts/snapshot-extract-benchmarkoor.sh + + # Only run compaction provided by the client. Other datadirs are exported + # after replay without external database tools or manual file replacement. + - name: Compact with client command + if: matrix.client == 'erigon' + run: sudo -E "$RUNNER_TEMP/benchmarkoor" db compact --client "$CLIENT" --datadir "$DATADIR" --image "$SNAPSHOT_IMAGE" + + - name: Report skipped compaction + if: matrix.client != 'erigon' + run: echo "Compaction skipped for $CLIENT; no client-owned compaction command is integrated." + + - name: Reopen Nethermind before export + if: matrix.client == 'nethermind' + run: .github/scripts/snapshot-reopen-nethermind.sh + + - name: Export + if: inputs.publish + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + RELEASE: ${{ inputs.release }} + EEST: ${{ needs.prepare.outputs.eest }} + BASE: ${{ needs.prepare.outputs.base }} + BUCKET: ${{ inputs.bucket }} + run: .github/scripts/snapshot-export.sh + + - name: Nethermind reopen evidence + if: always() && matrix.client == 'nethermind' + uses: actions/upload-artifact@v4 + with: + name: snapshot-nethermind-reopen + path: ${{ env.W }}/nethermind-reopen + if-no-files-found: ignore + + - name: Clean up + if: always() + run: | + docker rm -f "snapshot-nethermind-reopen-${GITHUB_RUN_ID}" >/dev/null 2>&1 || true + sudo -E "${BENCHMARKOOR_SCHELK_BIN:-schelk}" recover -y + sudo rm -rf "$RUNNER_TEMP/snapshot" + rm -f "$RUNNER_TEMP/benchmarkoor" + + # latest is written last and only once every client is up, so a consumer of it + # never meets a height some client lacks. + latest: + if: inputs.publish + needs: [prepare, geth, replay] + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.S3_ENDPOINT_URL }} + BUCKET: ${{ inputs.bucket }} + CLIENTS: ${{ inputs.clients }} + steps: + - uses: actions/download-artifact@v4 + with: + name: snapshot-tail + path: tail + + - name: Write latest + run: | + head=$(( $(jq -r .result.number tail/head.json) )) + clients=$(jq -r '["geth"] + . | unique | .[]' <<<"$CLIENTS") + for c in $clients; do + aws s3 ls "s3://$BUCKET/jochemnet/$c/$head/" | grep -q '\.tar\.zst$' || { echo "::error::no $c archive at $head"; exit 1; } + done + for c in $clients; do echo "$head" | aws s3 cp - "s3://$BUCKET/jochemnet/$c/latest"; done + echo "### jochemnet snapshot $head ($(jq -r .result.hash tail/head.json)): ${clients//$'\n'/ }" >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 2987db4..cb7b2bb 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ All workflows are `workflow_dispatch`-only and must be dispatched from the defau | `benchmarkoor-build-state-actor.yaml` | Build **only** the state-actor datadir (no payload fill). Has a `force` input to rebuild over an already-populated/partial datadir. | | `benchmarkoor-release.yaml` | Promote the filled payloads from a build run into a GitHub Release. | | `benchmarkoor-run.yaml` | Build the datadir, replay the payloads, and upload results to S3. | +| `benchmarkoor-snapshot.yaml` | Turn a jochemnet release into a shadowfork snapshot: every client's jochemnet datadir advanced by the release's pre-run, its gas limit walked down, compacted where supported, and published as `/jochemnet///`. See below. | Common inputs: `clients` (JSON array), `snapshot`, `context`, `subdir`, `test-type`, and `instance-id`. `benchmarkoor-build-state-actor.yaml` takes only `snapshot` + `clients` (plus `force`), since the test context is irrelevant to a datadir-only build. @@ -93,6 +94,29 @@ Config merge order: - **build:** `global` → `datadirs//{global,builder}` → `contexts///{global, test-source..builder}` - **run:** `global` → resource-limits → `s3-upload` → `datadirs//{global,runner}` → `contexts///{global, test-source..runner, clients}` +### Shadowfork snapshots (`benchmarkoor-snapshot.yaml`) + +A devnet that shadowforks jochemnet (msf-2) boots these datadirs as they are, so it gets the +release's contracts without a pre-run of its own. Given a release tag: + +1. **geth** (the pinned jochemnet geth host) replays the release's pre-run onto its snapshot and + walks the gas limit down to `gas-limit` with empty blocks, recorded as the *tail* bundle + (`configs/snapshots/jochemnet/v1/builder.yaml`, target `geth-ramp`). It runs with + `--cache.gc=0`, so no block outside geth's 128 diff layers is left in the journal. A restarted + geth must come back at the tail's head; its head block is what every client publishes. +2. **every other client** replays the release's pre-run and then the tail onto its own snapshot. + Every payload must be VALID, so every image agrees with geth at every block. +3. **Geth and Erigon** are compacted with their own commands (`benchmarkoor db compact`). + Compaction is skipped for the other clients; no external database tool or manual file swap + is used. Nethermind must reopen offline at the expected head before export. Each datadir + is then **exported** (`benchmarkoor snapshot export`). +4. `latest` is written last, once every client's archive is up. + +Runs on the jochemnet hosts, whose schelk baseline is the release's base snapshot (checked against +`configs/datadirs/jochemnet/v1/global.yaml`), and `schelk recover`s the scratch afterwards. +Publishing uses the repo's `S3_ENDPOINT_URL` / `AWS_*` secrets; the first job checks they can write +the bucket. Each host needs ~55 GB free for the release's pre-run bundle (50 GB uncompressed). + ## Dispatchoor The `dispatchoor/` directory holds generated job definitions for [dispatchoor](https://github.com/ethpandaops/dispatchoor) to trigger `benchmarkoor-run.yaml`. diff --git a/configs/snapshots/jochemnet/v1/builder.yaml b/configs/snapshots/jochemnet/v1/builder.yaml new file mode 100644 index 0000000..3ea7a98 --- /dev/null +++ b/configs/snapshots/jochemnet/v1/builder.yaml @@ -0,0 +1,121 @@ +# Pre-runs that turn a jochemnet release into a shadowfork snapshot +# (benchmarkoor-snapshot.yaml). Loaded after configs/datadirs/jochemnet/v1/global.yaml +# (snapshot dirs, genesis files) and the release's own context global.yaml +# (AMSTERDAM_ACTIVATION_TS), so the schedule is the one the release was built on. +# +# The workflow sets SNAPSHOT_BASE_BUNDLE (the release's pre_run_bundle), +# SNAPSHOT_TAIL_DIR (where geth-ramp writes its bundle, the one the replays +# continue with), SNAPSHOT_GAS_LIMIT and SNAPSHOT_IMAGE, and builds one target +# per host with --limit-pre-runs-target. +builder: + pre_runs: + pull_policy: always + config: + fork: amsterdam + datadir_method: schelk + targets: + # geth continues the release's pre-run and walks the gas limit down from the + # payloads' 1 TGas with empty blocks (~1/1024 per block, ~8.7k blocks to + # 200M). --cache.gc=0 leaves geth no write buffer, so every block that + # leaves the 128 diff layers is flushed to pebble: the journal holds only + # the last 128 ramp blocks, never the release's contracts. + - name: geth-ramp + filler_client: geth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${GETH_SNAPSHOT_DIR} + bundle_dir: ${SNAPSHOT_TAIL_DIR} + genesis: ${GETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + fill: false + gas_limit: ${SNAPSHOT_GAS_LIMIT} + gas_bump_max_blocks: 10000 + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + - --cache.gc=0 + + # A geth rebuild from an existing tail (a re-run that must land on the same + # head as the other clients): replays the release and the tail like the + # others, with the ramp's --cache.gc=0 so the journal stays at 128 layers. + - name: geth-replay + filler_client: geth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${GETH_SNAPSHOT_DIR} + genesis: ${GETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + - --cache.gc=0 + + # Every other client replays the release, then the ramp, onto its own + # jochemnet snapshot. + # Each payload must come back VALID, so each client re-derives geth's state + # root, receipts and block access list at every block. + - name: nethermind-replay + filler_client: nethermind + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${NETHERMIND_SNAPSHOT_DIR} + genesis: ${NETHERMIND_CHAINSPEC} + genesis_eip_override: + timestamp: ${AMSTERDAM_ACTIVATION_TS} + eips: [2780, 7708, 7778, 7843, 7928, 7954, 7976, 7981, 7997, 8024, 8037, 8038, 8246, 8282] + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + # The snapshot dir is the chain column dir (datadirs/jochemnet/v1/global.yaml). + - --Init.BaseDbPath=/data + - --FlatDb.Enabled=true + - --FlatDb.CompactionOffset=0 + # A distributable snapshot must reopen without replaying a 1024-block + # flat-state window. Persist finalized state during the snapshot build; + # these settings do not change the benchmark or consumer defaults. + - --FlatDb.CompactSize=1 + - --FlatDb.MinReorgDepth=0 + - --FlatDb.MaxReorgDepth=0 + - --FlatDb.EnableLongFinality=false + + - name: besu-replay + filler_client: besu + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${BESU_SNAPSHOT_DIR} + genesis: ${BESU_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + + - name: reth-replay + filler_client: reth + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${RETH_SNAPSHOT_DIR} + genesis: ${RETH_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + + - name: erigon-replay + filler_client: erigon + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${ERIGON_SNAPSHOT_DIR} + genesis: ${ERIGON_GENESIS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --override.amsterdam=${AMSTERDAM_ACTIVATION_TS} + + - name: ethrex-replay + filler_client: ethrex + filler_image: ${SNAPSHOT_IMAGE} + source_dir: ${ETHREX_SNAPSHOT_DIR} + genesis: ${ETHREX_GENESIS} + genesis_fork_override: + amsterdam: ${AMSTERDAM_ACTIVATION_TS} + base_bundle: ${SNAPSHOT_BASE_BUNDLE} + replay_from: ${SNAPSHOT_TAIL_DIR}/pre_run_bundle + filler_extra_args: + - --skip-genesis-validation