msf-2: mainnet shadowfork from the cleaned jochemnet release, Gloas at genesis

.gitignore

@@ -38,3 +38,4 @@ scripts/*.json
 
 # derived genesis artifacts (too large for git)
 network-configs/devnet-9/parsed/**
+__pycache__/

README.md

@@ -27,6 +27,7 @@ WIP | [devnet-8](https://notes.ethereum.org/@ethpandaops/glamsterdam-devnet-8)|
  🔴 | sepsf-1 (sepolia shadowfork)| [Network config](network-configs/sepsf-1)    | [🔗](ansible/inventories/sepsf-1) | [🔗](terraform/sepsf-1) | [🔗](kubernetes-archive/sepsf-1)
  On | [sepsf-2](https://dora.glamsterdam-sepsf-2.ethpandaops.io) (sepolia shadowfork)| [Network config](network-configs/sepsf-2)    | [🔗](ansible/inventories/sepsf-2) | [🔗](terraform/sepsf-2) | [🔗](kubernetes/sepsf-2)
 WIP | msf-1 (mainnet shadowfork)| -    | [🔗](ansible/inventories/msf-1) | [🔗](terraform/msf-1) | -
+WIP | msf-2 (mainnet shadowfork, jochemnet release, benchmarking)| -    | [🔗](ansible/inventories/msf-2) | [🔗](terraform/msf-2) | -
 # Development
 ## Version management for tools
 

ansible/inventories/msf-2/group_vars/all/00-defaults.yaml

@@ -0,0 +1,85 @@
+ansible_user: devops
+ansible_python_interpreter: /usr/bin/python3
+# Template name refers to the name of the devnet without number
+# Generally refer to the first word of the repository name
+# e.g. peerdas, or pectra
+devnet_name: glamsterdam
+
+# IPv6 Specific configuration
+global_ipv6_enabled: true
+
+# Per-service IPv6 offset within the host's assigned prefix.
+# DO /124 has only ::3..::f usable, so offsets must stay small.
+# Use end of range for static assignments, start will be auto assigned to remaining containers.
+service_ipv6_offsets:
+  nginx_proxy: 15     # :f
+  execution: 14       # :e
+  snooper_engine: 13  # :d
+  snooper_rpc: 12     # :c
+  beacon: 11          # :b
+  validator: 10       # :a
+
+# Derive host IPv6 prefix from the Terraform-provided `ipv6` host var.
+# Hetzner gives a /64, DO gives a /124.
+host_ipv6_prefix_length: "{{ '64' if cloud | default('') == 'hetzner' else '124' }}"
+host_ipv6_prefix: "{{ (ipv6 | default('')) | ansible.utils.ipsubnet(host_ipv6_prefix_length | int) if (ipv6 | default('')) | length > 0 else '' }}"
+
+######################################################
+##
+##          Role specific overwrites
+##
+######################################################
+
+# role: ethpandaops.general.bootstrap
+bootstrap_user: root
+bootstrap_default_user_authorized_keys_github:
+  - barnabasbusa
+  - parithosh
+  - samcm
+  - savid
+  - skylenet
+  - pk910
+  - mattevans
+  - qu0b
+
+# role: robertdebock.fail2ban
+fail2ban_loglevel: INFO
+fail2ban_logtarget: /var/log/fail2ban.log
+fail2ban_maxretry: 10
+fail2ban_bantime: 600
+fail2ban_ignoreips:
+  - "127.0.0.1/8 ::1"
+  - "{{ lookup('ansible.builtin.url', 'http://ifconfig.me/ip', split_lines=False) }}/32" # Avoid banning ourself
+
+
+# role: ethpandaops.general.docker_network
+docker_network_name: shared
+docker_network_enable_ipv6: "{{ global_ipv6_enabled | default(false) | bool and (host_ipv6_prefix | default('') | length > 0) }}"
+docker_network_driver_options: >-
+  {{ {'com.docker.network.bridge.gateway_mode_ipv6': 'routed'}
+     if (docker_network_enable_ipv6 | bool) else {} }}
+docker_network_ipam_config: >-
+  {{ [{'subnet': host_ipv6_prefix,
+       'gateway': host_ipv6_prefix | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address')}]
+     if (docker_network_enable_ipv6 | bool) else [] }}
+docker_network_ipv6_pool: >-
+  {{ (host_ipv6_prefix | ansible.utils.ipsubnet(124, 0))
+     if (docker_network_enable_ipv6 | bool) else '' }}
+
+docker_networks_shared:
+  - name: "{{ docker_network_name }}"
+
+# role: geerlingguy.docker
+docker_users:
+  - devops
+docker_daemon_options:
+  "log-driver": "json-file"
+  "log-opts":
+      "max-size": "500m"
+      "max-file": "8"
+
+# role: ethpandaops.general.prometheus
+prometheus_container_networks: "{{ docker_networks_shared }}"
+
+# role: ethpandaops.general.ethereum_node
+ethereum_node_images_always_pull: true

ansible/inventories/msf-2/group_vars/all/images.yaml

@@ -0,0 +1,68 @@
+---
+default_ethereum_client_images:
+  ### Consensus layer clients
+  lighthouse: registry-1.docker.io/sigp/lighthouse:v8.3.0-rc.0
+  lodestar: registry-1.docker.io/chainsafe/lodestar:v1.49.0
+  nimbus: registry-1.docker.io/statusim/nimbus-eth2:multiarch-v26.10.0
+  prysm: registry-1.docker.io/offchainlabs/prysm-beacon-chain:v7.2.1
+  prysm_validator: registry-1.docker.io/offchainlabs/prysm-validator:v7.2.1
+  teku: registry-1.docker.io/consensys/teku:26.9.1
+  grandine: registry-1.docker.io/sifrai/grandine:3.0.0-rc.0
+  ### Execution layer clients
+  besu: registry-1.docker.io/hyperledger/besu:26.9.0
+  geth: registry-1.docker.io/ethereum/client-go:v1.17.7
+  erigon: registry-1.docker.io/erigontech/erigon:v3.7.1
+  nethermind: registry-1.docker.io/nethermind/nethermind:2.1.0
+  reth: ghcr.io/paradigmxyz/reth:v2.7.0
+  nimbusel: registry-1.docker.io/statusim/nimbus-eth1:v0.4.2
+  ethrex: ghcr.io/lambdaclass/ethrex:29.0.1
+
+default_tooling_images:
+  mev_boost: registry-1.docker.io/ethpandaops/mev-boost:develop
+  mev_builder: registry-1.docker.io/ethpandaops/reth-rbuilder:develop
+  mev_relay: registry-1.docker.io/ethpandaops/mev-boost-relay:main
+  buildoor: registry-1.docker.io/ethpandaops/buildoor:main-latest
+  authenticatoor: registry-1.docker.io/ethpandaops/service-authenticatoor:main-latest
+  xatu_sentry: registry-1.docker.io/ethpandaops/xatu:1.24.1
+  xatu_cannon: registry-1.docker.io/ethpandaops/xatu:latest
+  xatu_mimicry: registry-1.docker.io/ethpandaops/xatu:latest
+  xatu_cl_mimicry: registry-1.docker.io/ethpandaops/xatu:latest
+  xatu_relay_monitor: registry-1.docker.io/ethpandaops/xatu:latest
+  ethereum_metrics_exporter: registry-1.docker.io/ethpandaops/ethereum-metrics-exporter:latest
+  tx_fuzz: registry-1.docker.io/ethpandaops/tx-fuzz:latest
+  forkmon: registry-1.docker.io/skylenet/nodemonitor:darkmode
+  forky: registry-1.docker.io/ethpandaops/forky:latest
+  fauceth: registry-1.docker.io/skylenet/fauceth:fix_fee_estimation
+  powfaucet: registry-1.docker.io/pk910/powfaucet:v2.5.1 # fixed client bundle + 250000 ethTxGasLimit default
+  homepage: registry-1.docker.io/ethpandaops/ethereum-testnet-homepage:latest
+  checkpointz: registry-1.docker.io/ethpandaops/checkpointz:gloas-latest
+  blockscout: registry-1.docker.io/blockscout/blockscout:latest
+  blockscout_frontend: ghcr.io/blockscout/frontend:latest
+  beacon_metrics_gazer: registry-1.docker.io/dapplion/beacon-metrics-gazer:latest
+  eth_fauceth: registry-1.docker.io/chainflag/eth-faucet:latest
+  blobscan: registry-1.docker.io/blossomlabs/blobscan:latest
+  blobscan_indexer: registry-1.docker.io/blossomlabs/blobscan-indexer:latest
+  dora: registry-1.docker.io/ethpandaops/dora:master-latest
+  dugtrio: registry-1.docker.io/ethpandaops/dugtrio:bbusa-endpoint-failover
+  ethereum_genesis_generator: registry-1.docker.io/ethpandaops/ethereum-genesis-generator:6.2.0
+  tracoor: registry-1.docker.io/ethpandaops/tracoor:0.0.2-glamsterdam-devnet-11 # pinned: floating tag + IfNotPresent won't re-pull
+  ncli: registry-1.docker.io/status-im/nimbus-eth2:unstable
+  lcli: registry-1.docker.io/ethpandaops/lighthouse:unstable
+  zcli: gloas
+  assertoor: registry-1.docker.io/ethpandaops/assertoor:master-latest
+  erpc: ghcr.io/erpc/erpc:latest
+  prometheus: registry-1.docker.io/prom/prometheus:v2.40.7
+  node_exporter: registry-1.docker.io/prom/node-exporter:v1.5.0
+  cl_bootnode: registry-1.docker.io/protolambda/eth2-bootnode:cleanup
+  json_rpc_snooper: registry-1.docker.io/ethpandaops/rpc-snooper:latest
+  nginx_proxy: registry-1.docker.io/library/nginx:alpine
+  nginx_proxy_gen: registry-1.docker.io/nginxproxy/docker-gen
+  nginx_proxy_acme: registry-1.docker.io/nginxproxy/acme-companion
+  nginx_proxy_cert_loader: registry-1.docker.io/ethpandaops/debian-docker:latest
+  nginx_proxy_cert_linker: registry-1.docker.io/nginxproxy/docker-gen
+  vector: registry-1.docker.io/timberio/vector:0.56.0-alpine
+  spamoor: registry-1.docker.io/ethpandaops/spamoor:master-latest
+  blobber: registry-1.docker.io/ethpandaops/blobber:latest
+  syncoor_web: ghcr.io/ethpandaops/syncoor-web:master
+  syncoor_server: ghcr.io/ethpandaops/syncoor:master
+  slashoor: registry-1.docker.io/ethpandaops/slashoor:master

ansible/inventories/msf-2/group_vars/besu.yaml

@@ -0,0 +1,102 @@
+---
+# role: ethpandaops.general.bootstrap
+besu_extra_github_users: []  # handles not yet in the coredevs registry
+bootstrap_default_user_authorized_keys_github_team_el: >-
+  {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/besu?format=txt') | select | list)
+     + besu_extra_github_users }}
+
+# role: ethpandaops.general.snapshot_fetcher
+snapshot_fetcher_out_dir: /data/besu
+
+# role: ethpandaops.general.ethereum_node
+ethereum_node_el: besu
+# role: ethpandaops.general.besu
+besu_container_name: execution
+besu_container_image: "{{ default_ethereum_client_images.besu }}"
+besu_container_env:
+  VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}"
+  VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}"
+  LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}"
+  BESU_OPTS: "-Xmx16g -XX:MaxHeapFreeRatio=30 -XX:MinHeapFreeRatio=10"
+besu_container_volumes:
+  - "{{ besu_datadir }}:/data"
+  - "{{ besu_auth_jwt_path }}:/execution-auth.jwt:ro"
+  - "{{ eth_testnet_config_dir }}:/network-config:ro"
+besu_container_command_extra_args:
+  - --genesis-file=/network-config/genesis.json
+  - --bootnodes={{ ethereum_el_bootnodes | join(',') }}
+  - --discovery-mode=V5
+  - --rpc-http-api=ADMIN,ETH,MINER,NET,TXPOOL,WEB3
+  # Restored jochemnet snapshot: nothing to sync, FULL just follows the CL from the
+  # snapshot head (fusaka shadowforks). The snapshot may come from another besu
+  # version, so skip the datadir version check.
+  - --sync-mode=FULL
+  - --version-compatibility-protection=false
+  # nimbus-el has no snapshot and snap-syncs from the restored nodes; besu serves snap
+  # only when asked (geth and nethermind do by default).
+  - --snapsync-server-enabled=true
+  - --data-storage-format=BONSAI
+  - --bonsai-limit-trie-logs-enabled=false
+  - --color-enabled=false
+  - --cache-last-blocks=2048
+  - --rpc-http-max-active-connections=1000
+  # devnet8 block gas limit is 200M; raise the RPC cap so eth_call/estimateGas can simulate full-block txs
+  - --rpc-gas-cap=200000000
+besu_container_pull: true
+
+prometheus_config: |
+  global:
+    scrape_interval: 12s
+    evaluation_interval: 30s
+    scrape_timeout: 10s
+    external_labels:
+      instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"
+      ip_address: "{{ ansible_host }}"
+      network: "{{ ethereum_network_name }}"
+      testnet: "{{ ethereum_network_name }}"
+      execution_client: "{{ ethereum_node_el }}"
+      consensus_client: "{{ ethereum_node_cl }}"
+      supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}"
+  remote_write:
+    - queue_config:
+        batch_send_deadline: 5s
+        max_backoff: 500ms
+        max_samples_per_send: 500
+        min_backoff: 50ms
+        max_shards: 100
+      url: {{ prometheus_remote_push_url }}
+      remote_timeout: 10s
+      basic_auth:
+        username: {{ prometheus_remote_write_username }}
+        password: {{ prometheus_remote_write_password }}
+  scrape_configs:
+    - job_name: "prometheus"
+      metrics_path: "/metrics"
+      static_configs:
+        - targets: ["localhost:9090"]
+          labels:
+            instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"
+    - job_name: "node"
+      metrics_path: "/metrics"
+      static_configs:
+        - targets: ["172.17.0.1:9100"]
+          labels:
+            instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"
+    - job_name: "exporter"
+      metrics_path: "/metrics"
+      static_configs:
+        - targets: ["ethereum-metrics-exporter:9090"]
+          labels:
+            instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"
+    - job_name: "consensus_node"
+      metrics_path: "/metrics"
+      static_configs:
+        - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"]
+          labels:
+            instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"
+    - job_name: "execution"
+      metrics_path: "/metrics"
+      static_configs:
+        - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"]
+          labels:
+            instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}"

ansible/inventories/msf-2/group_vars/blobber.yaml

@@ -0,0 +1,56 @@
+---
+blobber_container_image: "{{ default_tooling_images.blobber }}"
+
+blobber_log_level: trace
+
+ethereum_node_blobber_enabled: true
+
+# Storage configs
+blobber_datadir: /data/blobber
+blobber_validator_local_key_folder: "{{ ansible_inventory_sources[0] | dirname }}/files/validator_keys/{{ inventory_hostname }}"
+blobber_validator_remote_key_folder: "{{ blobber_datadir }}/validators"
+
+blobber_container_volumes:
+  - "{{ blobber_datadir }}:/data"
+
+# Blobber specific configs
+blobber_beacon_api: beacon:5052
+# blobber_slot_actions: {"name": "equivocating_blob_sidecars", "broadcast_blobs_first": true}
+# blobber_slot_actions: {"name": "conflicting_blobs"}
+blobber_slot_actions: { name: blob_gossip_delay, delay_milliseconds: 1500 }
+blobber_slot_action_frequency: 1
+
+blobber_container_ports:
+  - "{{ blobber_p2p_port }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port }}:{{ blobber_p2p_port }}/udp"
+  - "{{ blobber_p2p_port + 1 }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port + 1 }}:{{ blobber_p2p_port }}/udp"
+  - "{{ blobber_p2p_port + 2 }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port + 2 }}:{{ blobber_p2p_port }}/udp"
+  - "{{ blobber_p2p_port + 3 }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port + 3 }}:{{ blobber_p2p_port }}/udp"
+  - "{{ blobber_p2p_port + 4 }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port + 4 }}:{{ blobber_p2p_port }}/udp"
+  - "{{ blobber_p2p_port + 5 }}:{{ blobber_p2p_port }}"
+  - "{{ blobber_p2p_port + 5 }}:{{ blobber_p2p_port }}/udp"
+  - 127.0.0.1:{{ blobber_api_listen_port }}:{{ blobber_api_listen_port }}
+
+#  - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-geth-1.{{ network_server_subdomain }}"
+#  - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-besu-1.{{ network_server_subdomain }}"
+#  - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-erigon-1.{{ network_server_subdomain }}"
+#  - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-nethermind-1.{{ network_server_subdomain }}"
+#  - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-reth-1.{{ network_server_subdomain }}"
+blobber_container_command:
+  - --cl={{ blobber_beacon_api }}
+  - --beacon-port-start={{ blobber_p2p_port }}
+  - --log-level={{ blobber_log_level }}
+  - --external-ip=0.0.0.0
+  - --slot-action={{ blobber_slot_actions | to_json }}
+  - --slot-action-frequency={{ blobber_slot_action_frequency }}
+  - --validator-key-folder=/data/validators
+  - --validator-proxy-port-start={{ blobber_api_listen_port }}
+  - --enable-unsafe-mode
+
+blobber_container_command_extra_args: []
+# Default image pull policy
+blobber_container_pull: false