feat(snapshot): build shadowfork snapshots from a jochemnet release

.github/scripts/check-datadir-layout.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Fail before a client boots on a datadir laid out differently from what it is started with.
+# geth started with --datadir on a flat instance dir (chaindata/ at the top) still opens the
+# database, but looks for triedb/merkle.journal under <datadir>/geth/, discards the state it
+# holds, rewinds the head and truncates state history: the datadir is spoiled, not just unused.
+#   check-datadir-layout.sh <client> <datadir>
+set -euo pipefail
+c=${1:?client} d=${2:?datadir}
+need() { [ -e "$d/$1" ] || { echo "::error::$c datadir $d has no $1 (wrong layout)"; exit 1; }; }
+case $c in
+  geth) need geth/chaindata
+        [ ! -e "$d/chaindata" ] || { echo "::error::$d/chaindata: a flat instance dir, geth wants it under $d/geth/"; exit 1; } ;;
+  nethermind) need nethermind_db/mainnet/state ;;
+  besu) need database ;;
+  # jochemnet's reth v2 archive: static_files/, rocksdb/ and the MDBX db at the datadir root.
+  reth) need static_files; need rocksdb ;;
+  # erigon's pruned archive is its datadir: chaindata/ (mdbx) and snapshots/ (segments).
+  erigon) need chaindata; need snapshots ;;
+  # ethrex keeps its RocksDB at <datadir>/chain-1; the archive is that directory's contents.
+  ethrex) need chain-1/CURRENT ;;
+  *) echo "::error::unknown client $c"; exit 1 ;;
+esac
+echo "$c datadir layout ok: $d"

.github/scripts/release-pre-run-bundle.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Fetch a benchmarkoor-release's pre-run bundle into <dest>/pre_run_bundle, checked
+# against the release's sha256. Prints the bundle's pre-run.meta.json.
+#   release-pre-run-bundle.sh <release-tag> <dest>
+set -euo pipefail
+tag=${1:?release tag} dest=${2:?dest dir}
+base="https://github.com/${GITHUB_REPOSITORY:-ethpandaops/benchmarkoor-tests}/releases/download/$tag"
+dl="$dest/download"
+mkdir -p "$dl"
+
+curl -fsSL --retry 5 "$base/manifest.json" -o "$dl/manifest.json"
+# Every client's run replays the geth pre-run (test-source.*.runner.yaml), so it is the only one.
+asset=$(jq -ce '.assets[] | select(.client == "geth")' "$dl/manifest.json")
+tarball=$(jq -r .tarball <<<"$asset")
+mapfile -t files < <(jq -r '.files[]' <<<"$asset")
+
+for f in "${files[@]}"; do curl -fsSL --retry 5 "$base/$f" -o "$dl/$f"; done
+# Assets over 2 GB are split into .part-NNN; an unsplit asset is the tarball itself.
+[ "${#files[@]}" -eq 1 ] && [ "${files[0]}" = "$tarball" ] || { (cd "$dl" && cat "${files[@]}") > "$dl/$tarball"; }
+echo "$(jq -r .sha256 <<<"$asset")  $dl/$tarball" | sha256sum -c --quiet -
+
+tar -xzf "$dl/$tarball" -C "$dest" --strip-components=3 benchmarkoor-build-artifacts/pre-runs/geth/pre_run_bundle
+rm -rf "$dl"
+cat "$dest/pre_run_bundle/pre-run.meta.json"

.github/scripts/snapshot-export.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Export $DATADIR as <bucket>/jochemnet/$CLIENT/<head>/ with its metadata and head block
+# (benchmarkoor snapshot export). The head block is geth's, from the snapshot-tail artifact.
+# Env: CLIENT DATADIR SNAPSHOT_IMAGE SNAPSHOT_TAIL_DIR SNAPSHOT_GAS_LIMIT AMSTERDAM_TIME RELEASE EEST BASE BUCKET.
+# Production path matches the workflow latest job; scratch exports use the CLI directly.
+set -euo pipefail
+blk=$SNAPSHOT_TAIL_DIR/head.json
+meta=$SNAPSHOT_TAIL_DIR/pre_run_bundle/pre-run.meta.json
+head=$(( $(jq -r .result.number "$blk") ))
+
+jq -n --arg img "$SNAPSHOT_IMAGE" --arg release "$RELEASE" --arg eest "$EEST" --argjson base "$BASE" \
+  --argjson head "$head" --arg hash "$(jq -r .result.hash "$blk")" --argjson ams "$AMSTERDAM_TIME" \
+  --argjson gas "$SNAPSHOT_GAS_LIMIT" --argjson ramp "$(jq .payloads "$meta")" \
+  --arg run "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" '{
+    docker_image: $img,
+    shadowfork: {network: "jochemnet", base: $base, release: $release, eest: $eest,
+      head: $head, head_hash: $hash, gas_limit: $gas, ramp_blocks: $ramp, amsterdam_time: $ams,
+      filler: "geth", built_by: $run,
+      notes: "already on Amsterdam: a devnet must start on Gloas with amsterdamTime = amsterdam_time"}}' \
+  > "$RUNNER_TEMP/snapshot/metadata.json"
+
+sudo -E "$RUNNER_TEMP/benchmarkoor" snapshot export --client "$CLIENT" --datadir "$DATADIR" \
+  --network jochemnet --block "$head" --bucket "$BUCKET" \
+  --head-block-file "$blk" --metadata-file "$RUNNER_TEMP/snapshot/metadata.json"

.github/scripts/snapshot-extract-benchmarkoor.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# The benchmarkoor action deletes its binary. Restore it from the same local
+# image for the subsequent compaction/export steps; do not pull a moving tag again.
+# Env: RUNNER_TEMP, BENCHMARKOOR_IMAGE (empty selects the action's local build).
+set -euo pipefail
+image=${BENCHMARKOOR_IMAGE:-benchmarkoor:local}
+image_id=$(docker image inspect --format '{{.Id}}' "$image")
+cid=$(docker create "$image_id")
+trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT
+bin=${RUNNER_TEMP:?runner temp}/benchmarkoor
+docker cp "$cid:/usr/local/bin/benchmarkoor" "$bin"
+chmod +x "$bin"
+"$bin" db compact --help >/dev/null
+"$bin" snapshot export --help >/dev/null

.github/scripts/snapshot-reopen-nethermind.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Reopen the stopped snapshot with the consumer chainspec before publication.
+# Env: DATADIR, SNAPSHOT_IMAGE, SNAPSHOT_TAIL_DIR, AMSTERDAM_TIME, W.
+set -euo pipefail
+DATADIR=${DATADIR:?datadir}; SNAPSHOT_IMAGE=${SNAPSHOT_IMAGE:?image}
+GENESIS=${W:?work directory}/nethermind-reopen-genesis.json
+OUT=$W/nethermind-reopen
+expected=${SNAPSHOT_TAIL_DIR:?tail directory}/head.json
+name=snapshot-nethermind-reopen-${GITHUB_RUN_ID:-$$}
+url=$(sed -n 's/^ *GETH_GENESIS: *//p' configs/datadirs/jochemnet/v1/global.yaml)
+[ -n "$url" ]
+# Geth permits unprefixed alloc addresses; Nethermind's genesis reader requires 0x.
+curl -fsSL --max-time 60 "$url" | jq --argjson ams "${AMSTERDAM_TIME:?fork timestamp}" '
+  .config.amsterdamTime = $ams |
+  .alloc |= with_entries(.key |= if startswith("0x") then . else "0x" + . end)
+' > "$GENESIS"
+mkdir -p "$OUT"
+rm -f "$OUT/PASS"
+want=$(jq -er .result.hash "$expected")
+number=$(jq -er .result.number "$expected")
+state_root=$(jq -er .result.stateRoot "$expected")
+DBREL=nethermind_db/mainnet
+[ ! -d "$DATADIR/mainnet" ] || DBREL=mainnet
+[ -d "$DATADIR/$DBREL" ]
+[ "$(jq -r .config.amsterdamTime "$GENESIS")" = "$AMSTERDAM_TIME" ]
+[ -z "$(docker ps -aq --filter "name=^${name}$")" ]
+cleanup() {
+  docker stop -t 600 "$name" > "$OUT/stop.log" 2>&1 || true
+  docker logs "$name" > "$OUT/container.log" 2>&1 || true
+  docker inspect -f '{{.State.ExitCode}}' "$name" > "$OUT/exit-code" 2>/dev/null || true
+  docker rm "$name" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+docker run -d --name "$name" --user 0:0 --network host \
+  -v "$DATADIR:/data" -v "$GENESIS:/network-genesis.json:ro" "$SNAPSHOT_IMAGE" \
+  --config=mainnet --datadir=/data --Init.ChainSpecPath=/network-genesis.json --Merge.TerminalTotalDifficulty=0 \
+  --Init.BaseDbPath="/data/$DBREL" --FlatDb.Enabled=true \
+  --Network.MaxActivePeers=0 --Init.DiscoveryEnabled=false \
+  --Network.DiscoveryPort=30399 --Network.P2PPort=30399 \
+  --JsonRpc.Enabled=true --JsonRpc.Host=127.0.0.1 --JsonRpc.Port=18545 \
+  --JsonRpc.EngineHost=127.0.0.1 --JsonRpc.EnginePort=18551 \
+  --Sync.NetworkingEnabled=false --Sync.FastSync=false --Sync.SnapSync=false
+for ((n=0; n<180; n++)); do
+  if curl -fsS --max-time 5 -H 'Content-Type: application/json' \
+    --data '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}' \
+    http://127.0.0.1:18545 > "$OUT/head.tmp.json" 2>/dev/null; then
+    mv "$OUT/head.tmp.json" "$OUT/head.json"
+    jq -c '.result | {number,hash,stateRoot}' "$OUT/head.json"
+    if jq -e --arg want "$want" --arg number "$number" --arg root "$state_root" '.result.hash == $want and .result.number == $number and .result.stateRoot == $root' "$OUT/head.json" >/dev/null; then
+      curl -fsS --max-time 10 -H 'Content-Type: application/json' \
+        --data '{"jsonrpc":"2.0","id":1,"method":"eth_config","params":[]}' \
+        http://127.0.0.1:18545 > "$OUT/eth-config.json"
+      activation=$(jq -er '.result.current.activationTime' "$OUT/eth-config.json")
+      [[ "$activation" =~ ^(0x[0-9a-fA-F]+|[0-9]+)$ ]]
+      [ "$((activation))" -eq "$AMSTERDAM_TIME" ]
+      for addr in 0x0000bff46984e3725691fa540a8c7589300d8282 0x000064d678505ad48f8ccb093bc65613800e8282; do
+        curl -fsS --max-time 10 -H 'Content-Type: application/json' \
+          --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getCode\",\"params\":[\"$addr\",\"latest\"]}" \
+          http://127.0.0.1:18545 > "$OUT/code-$addr.json"
+        jq -e '.result | type == "string" and length > 2' "$OUT/code-$addr.json" >/dev/null
+      done
+      cleanup
+      trap - EXIT
+      case "$(cat "$OUT/exit-code")" in 0|130|143) ;; *) exit 1 ;; esac
+      date -u +%FT%TZ > "$OUT/PASS"
+      echo 'PASS: Nethermind reopens offline at the snapshot head'
+      exit 0
+    fi
+  fi
+  [ "$(docker inspect -f '{{.State.Running}}' "$name")" = true ] || break
+  sleep 5
+done
+echo 'FAIL: Nethermind did not reopen at the required head' >&2
+exit 1