From b03267c3082ad3b8d883a22b175774713495b6f8 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 10:21:56 +0200 Subject: [PATCH 1/2] msf-2: copy msf-1 deployment as the review baseline Mechanical copy of the msf-1 deployment at 61aefb8 (#92), with network names/URLs and the state key renamed to msf-2, VPC changed from 10.32 to 10.33, and encrypted metrics/log identities renamed. Exclude msf-1's generated inventory and chain-specific prestate outputs. Preserve the original copy from c8a5327; all functional msf-2 changes are isolated in the following commit. --- README.md | 1 + .../msf-2/group_vars/all/00-defaults.yaml | 85 +++ .../msf-2/group_vars/all/all.sops.yaml | 203 ++++++ .../inventories/msf-2/group_vars/all/all.yaml | 598 ++++++++++++++++++ .../msf-2/group_vars/all/images.yaml | 68 ++ .../inventories/msf-2/group_vars/besu.yaml | 102 +++ .../inventories/msf-2/group_vars/blobber.yaml | 56 ++ .../msf-2/group_vars/bootnode.sops.yaml | 158 +++++ .../msf-2/group_vars/bootnode.yaml | 181 ++++++ .../msf-2/group_vars/buildoor.yaml | 83 +++ .../msf-2/group_vars/dns_server.yaml | 93 +++ .../inventories/msf-2/group_vars/erigon.yaml | 113 ++++ .../msf-2/group_vars/ethereum_node.yaml | 118 ++++ .../inventories/msf-2/group_vars/ethrex.yaml | 98 +++ .../inventories/msf-2/group_vars/geth.yaml | 93 +++ .../msf-2/group_vars/grandine.yaml | 55 ++ .../msf-2/group_vars/lighthouse.yaml | 53 ++ .../msf-2/group_vars/lodestar.yaml | 55 ++ .../msf-2/group_vars/nethermind.yaml | 106 ++++ .../inventories/msf-2/group_vars/nimbus.yaml | 65 ++ .../msf-2/group_vars/nimbusel.yaml | 99 +++ .../inventories/msf-2/group_vars/prysm.yaml | 65 ++ .../inventories/msf-2/group_vars/reth.yaml | 91 +++ .../inventories/msf-2/group_vars/teku.yaml | 56 ++ .../msf-2/group_vars/tx_fuzz_blobs.yaml | 13 + .../msf-2/group_vars/tx_fuzz_txs.yaml | 12 + .../inventories/msf-2/group_vars/xatu.yaml | 91 +++ .../msf-2/group_vars/xatu_sentry.yaml | 2 + .../host_vars/buildoor-teku-nethermind-1.yaml | 4 + ansible/inventories/msf-2/host_vars/localhost | 2 + terraform/msf-2/ansible_inventory.tmpl | 122 ++++ terraform/msf-2/cloudflare.tf | 60 ++ terraform/msf-2/digitalocean.tf | 181 ++++++ terraform/msf-2/firewall.tf | 370 +++++++++++ terraform/msf-2/hetzner.tf | 220 +++++++ terraform/msf-2/main.tf | 99 +++ terraform/msf-2/nodes.tf | 113 ++++ terraform/msf-2/outputs.tf | 120 ++++ terraform/msf-2/ssh_config.tmpl | 16 + 39 files changed, 4120 insertions(+) create mode 100644 ansible/inventories/msf-2/group_vars/all/00-defaults.yaml create mode 100644 ansible/inventories/msf-2/group_vars/all/all.sops.yaml create mode 100644 ansible/inventories/msf-2/group_vars/all/all.yaml create mode 100644 ansible/inventories/msf-2/group_vars/all/images.yaml create mode 100644 ansible/inventories/msf-2/group_vars/besu.yaml create mode 100644 ansible/inventories/msf-2/group_vars/blobber.yaml create mode 100644 ansible/inventories/msf-2/group_vars/bootnode.sops.yaml create mode 100644 ansible/inventories/msf-2/group_vars/bootnode.yaml create mode 100644 ansible/inventories/msf-2/group_vars/buildoor.yaml create mode 100644 ansible/inventories/msf-2/group_vars/dns_server.yaml create mode 100644 ansible/inventories/msf-2/group_vars/erigon.yaml create mode 100644 ansible/inventories/msf-2/group_vars/ethereum_node.yaml create mode 100644 ansible/inventories/msf-2/group_vars/ethrex.yaml create mode 100644 ansible/inventories/msf-2/group_vars/geth.yaml create mode 100644 ansible/inventories/msf-2/group_vars/grandine.yaml create mode 100644 ansible/inventories/msf-2/group_vars/lighthouse.yaml create mode 100644 ansible/inventories/msf-2/group_vars/lodestar.yaml create mode 100644 ansible/inventories/msf-2/group_vars/nethermind.yaml create mode 100644 ansible/inventories/msf-2/group_vars/nimbus.yaml create mode 100644 ansible/inventories/msf-2/group_vars/nimbusel.yaml create mode 100644 ansible/inventories/msf-2/group_vars/prysm.yaml create mode 100644 ansible/inventories/msf-2/group_vars/reth.yaml create mode 100644 ansible/inventories/msf-2/group_vars/teku.yaml create mode 100644 ansible/inventories/msf-2/group_vars/tx_fuzz_blobs.yaml create mode 100644 ansible/inventories/msf-2/group_vars/tx_fuzz_txs.yaml create mode 100644 ansible/inventories/msf-2/group_vars/xatu.yaml create mode 100644 ansible/inventories/msf-2/group_vars/xatu_sentry.yaml create mode 100644 ansible/inventories/msf-2/host_vars/buildoor-teku-nethermind-1.yaml create mode 100644 ansible/inventories/msf-2/host_vars/localhost create mode 100644 terraform/msf-2/ansible_inventory.tmpl create mode 100644 terraform/msf-2/cloudflare.tf create mode 100644 terraform/msf-2/digitalocean.tf create mode 100644 terraform/msf-2/firewall.tf create mode 100644 terraform/msf-2/hetzner.tf create mode 100644 terraform/msf-2/main.tf create mode 100644 terraform/msf-2/nodes.tf create mode 100644 terraform/msf-2/outputs.tf create mode 100644 terraform/msf-2/ssh_config.tmpl diff --git a/README.md b/README.md index b2a9499c..0a245820 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ WIP | [devnet-8](https://notes.ethereum.org/@ethpandaops/glamsterdam-devnet-8)| 🔴 | sepsf-1 (sepolia shadowfork)| [Network config](network-configs/sepsf-1) | [🔗](ansible/inventories/sepsf-1) | [🔗](terraform/sepsf-1) | [🔗](kubernetes-archive/sepsf-1) On | [sepsf-2](https://dora.glamsterdam-sepsf-2.ethpandaops.io) (sepolia shadowfork)| [Network config](network-configs/sepsf-2) | [🔗](ansible/inventories/sepsf-2) | [🔗](terraform/sepsf-2) | [🔗](kubernetes/sepsf-2) 🔴 | msf-1 (mainnet shadowfork)| [Network config](network-configs/msf-1) | [🔗](ansible/inventories/msf-1) | [🔗](terraform/msf-1) | [🔗](kubernetes-archive/msf-1) +WIP | msf-2 (mainnet shadowfork, jochemnet release, benchmarking)| - | [🔗](ansible/inventories/msf-2) | [🔗](terraform/msf-2) | - # Development ## Version management for tools diff --git a/ansible/inventories/msf-2/group_vars/all/00-defaults.yaml b/ansible/inventories/msf-2/group_vars/all/00-defaults.yaml new file mode 100644 index 00000000..ea35be37 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/all/00-defaults.yaml @@ -0,0 +1,85 @@ +ansible_user: devops +ansible_python_interpreter: /usr/bin/python3 +# Template name refers to the name of the devnet without number +# Generally refer to the first word of the repository name +# e.g. peerdas, or pectra +devnet_name: glamsterdam + +# IPv6 Specific configuration +global_ipv6_enabled: true + +# Per-service IPv6 offset within the host's assigned prefix. +# DO /124 has only ::3..::f usable, so offsets must stay small. +# Use end of range for static assignments, start will be auto assigned to remaining containers. +service_ipv6_offsets: + nginx_proxy: 15 # :f + execution: 14 # :e + snooper_engine: 13 # :d + snooper_rpc: 12 # :c + beacon: 11 # :b + validator: 10 # :a + +# Derive host IPv6 prefix from the Terraform-provided `ipv6` host var. +# Hetzner gives a /64, DO gives a /124. +host_ipv6_prefix_length: "{{ '64' if cloud | default('') == 'hetzner' else '124' }}" +host_ipv6_prefix: "{{ (ipv6 | default('')) | ansible.utils.ipsubnet(host_ipv6_prefix_length | int) if (ipv6 | default('')) | length > 0 else '' }}" + +###################################################### +## +## Role specific overwrites +## +###################################################### + +# role: ethpandaops.general.bootstrap +bootstrap_user: root +bootstrap_default_user_authorized_keys_github: + - barnabasbusa + - parithosh + - samcm + - savid + - skylenet + - pk910 + - mattevans + - qu0b + +# role: robertdebock.fail2ban +fail2ban_loglevel: INFO +fail2ban_logtarget: /var/log/fail2ban.log +fail2ban_maxretry: 10 +fail2ban_bantime: 600 +fail2ban_ignoreips: + - "127.0.0.1/8 ::1" + - "{{ lookup('ansible.builtin.url', 'http://ifconfig.me/ip', split_lines=False) }}/32" # Avoid banning ourself + + +# role: ethpandaops.general.docker_network +docker_network_name: shared +docker_network_enable_ipv6: "{{ global_ipv6_enabled | default(false) | bool and (host_ipv6_prefix | default('') | length > 0) }}" +docker_network_driver_options: >- + {{ {'com.docker.network.bridge.gateway_mode_ipv6': 'routed'} + if (docker_network_enable_ipv6 | bool) else {} }} +docker_network_ipam_config: >- + {{ [{'subnet': host_ipv6_prefix, + 'gateway': host_ipv6_prefix | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address')}] + if (docker_network_enable_ipv6 | bool) else [] }} +docker_network_ipv6_pool: >- + {{ (host_ipv6_prefix | ansible.utils.ipsubnet(124, 0)) + if (docker_network_enable_ipv6 | bool) else '' }} + +docker_networks_shared: + - name: "{{ docker_network_name }}" + +# role: geerlingguy.docker +docker_users: + - devops +docker_daemon_options: + "log-driver": "json-file" + "log-opts": + "max-size": "500m" + "max-file": "8" + +# role: ethpandaops.general.prometheus +prometheus_container_networks: "{{ docker_networks_shared }}" + +# role: ethpandaops.general.ethereum_node +ethereum_node_images_always_pull: true diff --git a/ansible/inventories/msf-2/group_vars/all/all.sops.yaml b/ansible/inventories/msf-2/group_vars/all/all.sops.yaml new file mode 100644 index 00000000..9d761b56 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/all/all.sops.yaml @@ -0,0 +1,203 @@ +secret_zerossl: + ACME_EAB_KID: ENC[AES256_GCM,data:UAbtTHRnXHispR1iQr/5eQMV7JMk5g==,iv:WkDvniP4a1cjpRTQqT0MnJo9UkIp+iwJ3NPSTb8sFsU=,tag:yXavQCYBVJsPHwmm2WbwSg==,type:str] + ACME_EAB_HMAC_KEY: ENC[AES256_GCM,data:V7U8TluhY0y550IjMfubUgHSFCdxqS109TwWjSGx818J2So5zrTJR0TWzRl1bH4/9MyqCpEtTLHzLlL0j7I4XzgNvrm8qjpeNw3yRa94eyWH5EJT8VI=,iv:Evd5W8ORLtUa3sSl6XeaqY69p0FAmpiFmjXDlP+XbUY=,tag:A34c1/jnfKSFjuDf/2stYA==,type:str] +secret_prometheus_remote_write: + username: ENC[AES256_GCM,data:sLeD3W2HobqS99rstG9qukc=,iv:H5QuY4h5RZxcoBg6qOR/uaWGeHsEDmQJ8uipsi1GF0Q=,tag:CiXtlhhH6/Jz/n2k2E+NFQ==,type:str] + password: ENC[AES256_GCM,data:FsxhGsp3XUSWEZbJVYFv2V6L5HYEv6mNSfxdRWPZSkOvTG4B,iv:JkK+me7CBgP44gwHwZdCfpd5ZKhUmG9Pr7uPelW0JuM=,tag:LjbmI9MHSeyLkPl1Vg5Nyw==,type:str] +secret_loki: + endpoint: ENC[AES256_GCM,data:fgNKy2JFJ+eWraqSN8mmrMD2qua/od9Ai4KkWt0BaqJuJeNbkisVERcjZXL/OFZK0eJqNVzQh+w=,iv:3hdFH0Y+33EXpCMoEhQHMj1cwxn2EMFOdPj2YwJZAhg=,tag:H6KbRsCGIC0axq4qFfP++A==,type:str] + username: ENC[AES256_GCM,data:hX2WNfHo3UumyB9y7ti+9JY=,iv:zRvSJabBqxANzCOixupm9ghIMriKQTrqlzkLAFy22fk=,tag:umcsnxdjasOYZ6achgw0kw==,type:str] + password: ENC[AES256_GCM,data:Lup8xKhAXpbSjGU1aICYM4CR0jHzm0FYSz4a38H/psHLSHln,iv:576Quu09HRoruCzO9l1j6b9wlO0er/NKe1HTxNJudzc=,tag:+Vb16lFEFYR4JfpvOYZV1w==,type:str] +secret_nginx_shared_basic_auth: + name: ENC[AES256_GCM,data:O7L3,iv:DiOGlqfOfrDlt7X4OGY27OYlkDDEHguv+kg1zRhBek0=,tag:KuOX20lI/iYZOORFraiISw==,type:str] + password: ENC[AES256_GCM,data:HyMDeoK5s6JDfsJ7j5Gg,iv:yvC4WWOOJvVeegWl0uj2P8yxcfBgNrvYSm5xUB5qux8=,tag:XoZbXpvLtUJPVoEaV3ltsg==,type:str] +secret_ethstats: ENC[AES256_GCM,data:vO/gY2iluciwksE=,iv:WHYxXgQ2LdLGMKxvagmT3UhmQl/dRucpyhYzZxHvLHc=,tag:NX4Lkg6SPPusRx/zHGn00w==,type:str] +secret_bootnodoor_seed: ENC[AES256_GCM,data:A611+/dn9lT08iGrqgLNgqPpWGEyGf1usFFWM4P9uB0Twfryjk5bzBVFLcLPKzkhuj4DThToS8F3s4iCAmq/+Q==,iv:ic2cap4ynWi/IS6dkn3fkKgNTndnZCxTbBbIABpXgrA=,tag:1Jwqi77lNG3yXZ9RZnZe5g==,type:str] +secret_genesis_mnemonic: ENC[AES256_GCM,data:zRxUC65Kt1b8DM4QPyEJ/u0Uhp2ftMcJVtacbXRrOCUGm+gy0n+VH199hVxvviouJQkbv5FiJYiFjwjJ/yFX5ajPgOqanjRpBfEfwwD2Drtafaw/mWT7pSILHlo7oI18+bGIWjXve5mpUSuAC7jiaufDVROvSa6bcew0Ogbp5pzbFJBFFim3LFjyC1uSRO7JbRoK4cUl1CxnaCUaC/3GKz4l/fIe,iv:EaQilsQbnswzCqkWSK9G/R3wwmMRDwWZDjdZQ1fUZRk=,tag:2kWU1BNG/B82qzmNxmwIjQ==,type:str] +secret_mev_coinbase_secret_key: ENC[AES256_GCM,data:TYLALYjoxOyyrpPoJ/gBvXO2vMdbbFqrI0gyZAJ7MHgI7SMmb7qTfDzakHmOASZ5ezJXfOMWqy0zBqQwymLhzA==,iv:MNJfTjd3pfAW9tR8WUEcp5BOcjjBTQFRoAN+NkC+VAA=,tag:dlR3sMxJb9he8xZn3FcD6w==,type:str] +secret_mev_builder_tx_signing_key: ENC[AES256_GCM,data:Tr01nA3sls3AhttJga/ndK+nMjZSiyMIE3zafwsEZjZt9aETG+zEnkcAK5y9P1aq2N1UZ/KMOF0BPNbgCtOddw==,iv:tBmNG6Esy/3HFCiNZIggEb2Xlgc5MEwS4mVgQpcuSyc=,tag:2mfyvSzaMvCqeIFQGV7NMA==,type:str] +secret_mev_optimistic_relay_secret_key: ENC[AES256_GCM,data:S+hUZla9PQRe22mOoT4qy839Slvej86L2SduROkh3JlMDlnQDtAODv1nRI48JaXT2pBrr5cK7zYSaFICuU6+3w==,iv:9LupHVxZ+DMqY4ZAV8tGcjUuXSvjW8aroJ6HG8psauk=,tag:acL6x1VMrbh0XbXtrQk++g==,type:str] +secret_mev_relay_secret_key: ENC[AES256_GCM,data:jnk2Bp3kMu4XBAfe3cJ8iJ061Lq2UDoE5OLiGss1kmmh1W9x/7PVDRH/y6ysL2kS8vbIlTNPtsK8oRQk7MMXLQ==,iv:2QmzoczmbyFyB4UnmvoZDeAEXjIzXnFBY+acmVA6Ins=,tag:RwjvzIB8l2V5F912IAicvQ==,type:str] +secret_mev_flood_private_key: ENC[AES256_GCM,data:LYJDMo2JUMH5TnRJ/DiTpeF+u5oKTRJzpLwNte1QA84e8JNtKlehUDH5XX+rzoyOI3edHm4gwZ5oIeziY1STJA==,iv:1qNfUVqJ9eebRmA4Ly59KFR9WmaQ8eelSvNLJAWCJ10=,tag:2YTI3l+1z9sO2Gr7Csd4aw==,type:str] +secret_mev_flood_user_key: ENC[AES256_GCM,data:KKnBt1BEnBR/Cl9B7FljzV3kJOW9Gmp7DJ46xc1EkUZmWn1f8F4QLhgsyK34FKBKSCmDMCQoctwirAKTG/Vlsw==,iv:efkOBcrGeGJwT91SR+2wFI/fpnWASQeqjH3MJSIQBAA=,tag:1f+bkUNLh8kMsxO5qNQo+Q==,type:str] +secret_xatu_sentry: + server_address: ENC[AES256_GCM,data:eaGDu9nR3Aak04OF+QEi8AC8F0jq+RjKPe91ULpiIs8+ODeX8HUxl2rsMCrf4SNe,iv:Jx7ZOOYvVywAISapinvztKZT5PFrDkNqHiNKaDJbuIM=,tag:VGCX0e1VbKeBCixafLR8uQ==,type:str] + user: ENC[AES256_GCM,data:uwGLpiNh7Q==,iv:1beJ8Z7mcZzY55qRoYeoOHLluwAOVb4woG7vqF22x/w=,tag:aP85BC6u1T5dDETRRj569g==,type:str] + password: ENC[AES256_GCM,data:XCMEbRpVmRFaI10es9mBrR8EiFWJenK3zjM4QGh70iLo8HaU,iv:++cb9FjM21r9gylwvA5ax1E0FAQzZisAsJhGnPjQHdM=,tag:V5iTEAQ4JiUiiYeQqbMtDQ==,type:str] + event_ingester_auth: ENC[AES256_GCM,data:xD1uqYrRdB+TU7Fqbs+fENWiInVD1HOdVjxppQHhh7jRl+dU9HlJpD60nswd611EyHbWheTnlY1lRMO6Damz0ut9,iv:bKNxaPz+Xj5bIidQlaJrQXaDtT/I0lfBBY3zFfvMCAg=,tag:G1qOLehH6Wm9aI6a2ShqCQ==,type:str] + coordinator_secret: ENC[AES256_GCM,data:f4y7Ck0Z2zpOo8bHuS/w7Bhk6/6+DIvFXctMhjaMBYWLiwEY,iv:gQG7HBaDQPz0huJH2fq4y8HOGN3JwceJET/spgL5GXk=,tag:2guBo0MRKOMKOhpyjYL0VA==,type:str] +secret_cert_encryption_psk: ENC[AES256_GCM,data:WXK5OSHqbAPXxWRwxDV4vSkWLd+KBMJMZuEPFUKE,iv:w45No4rBDjT+kGooIoHGikjJQjwWQ9dyNBzL/NBo7XE=,tag:9Unr8yDbWpCOINcCW/frSg==,type:str] +secret_dora_api_key: ENC[AES256_GCM,data:RGBJ+xkv1WscCVO/Rd9/XZdOZatYCtSAKm/8yKVkwVr+fu0EB9LY7EG8fkw=,iv:YkgdX3hW+P53br6/uiNmmq4/qAyQ0/QrXSPgOjaMbVY=,tag:b/B9OBzYyWfJ5NemKpv2eQ==,type:str] +tx_fuzz_blobs_privkey: ENC[AES256_GCM,data:lYDYBAfPdtgovoGtmz7c4lUfiqxr2N0Sp1dkMWn115+TJYBzfpsMdfBl88PfAibFfHwhMhKwpvKTyHSeUzX2xw==,iv:qTk22/lpJyWFycayBupQp0sBaw2E2oq7peWypQh+0Ic=,tag:JhQPq8s3ueLEfL2FJGH+wA==,type:str] +tx_fuzz_txs_privkey: ENC[AES256_GCM,data:5oacWB7naaDLkIihcBK8E2AKPk9OWLz0AJ7ZLKmpTQkstF/idYkjm6GzmaX0LVGxND0DlUO/fcydtY+UFVQoyw==,iv:gHO+ttqzJCbRBCUvmlsTVNQK6vpnGXamIag63HS/fas=,tag:Ub7xqXrx9tYU2ytk/XtJCQ==,type:str] +goomy_private_key: ENC[AES256_GCM,data:cpKpCZPZOZ/oA4zSFJaIutD/H5SUOHLBS/qyKL+5ZOMOvaDVmI3zY3GiQl149LB3tya6Rg2rEQSABxR1fz8c5w==,iv:Ibm4sLvU+tNk7EqaSPs/2CRLU1yLlNDgSWjR+uNPrvQ=,tag:sOvyhcNsNv2Zj/7waeZBBA==,type:str] +nethermind_seq_api_key: ENC[AES256_GCM,data:C6dHJX02l+m0zVYH691GUyKZgRs=,iv:tNCA6W6hJm1PybTE2piAhfUjDKCqQWeN2FQfgiayJC0=,tag:VZiwt+DgR+nfWXdzZjGnxw==,type:str] +nethermind_seq_server: ENC[AES256_GCM,data:wEIM2pUqK+I3rf42maFTr05r8cOJWjnpPQ==,iv:2Iu5nQAjRmxqkOF2Pueup+Kiwd34sfTxWY8aFv0pea0=,tag:VrfL6pUITsKJPgSue/XXJw==,type:str] +nethermind_push_gateway: ENC[AES256_GCM,data:ScReyCHKHx2/8Yo3epSAsb3MykmlIrGkUegaV5+21aboPCgU+VbUOBfvqaTWpcQN3OjXH+euQj1bHL1etGk1qvjzm2sp/P9oqbHRU827nbejkk5SKEeC4q79U7QyQTbUJ1Z9paX1sYZysMDUf2LUb9r/wHhB7bknQYKnHBAg0Li7qw5T5o1yatJoDqfahe4R1Q==,iv:WuQDvBvS2zjEB805HYkq5TQ/H8baF23vyMP9JbdqCao=,tag:jTOnfh2+ZmWIXQBzIpIVCQ==,type:str] +tysm_secret_key: ENC[AES256_GCM,data:prgqa6JR5b18x6rvlrKs34KVeElIOU+JFTeRodYoaBXvddrJKsGmyBGPcFBdWsdhaQQNm9wbJuWnEIH50jB4KA==,iv:Z9ZzfhvAAxD0Bk6YxGTGHLlIyMUUneYyW5nBHj/j4rc=,tag:g7GqKhwMgyXGIWL1Lv2H3Q==,type:str] +tempo_grpc_url: ENC[AES256_GCM,data:ltAVTGgrqhUBXdAZe7D1HvdXK72YIORL/x4DYHgX911s+X8IZM9/guRqE1I/ZYSzNrQX0qON3/TrNSjpG1BUpkK9M0SLzqE4EKhaOOQonJRLunnufZVrZIDhXSMaGQhZcjsHQCV8,iv:4mzqA4Ck1g91+tST5oTSnTepikjOCWKJrV04Rsp/8Ts=,tag:MgjQUb+lR1SIU2d8KpvOew==,type:str] +secret_buildoor_wallet_privkey: ENC[AES256_GCM,data:MA5s/epX0pOuU/EsKlc3oXdo/jdjGazQB8aV50ANsIsiMqRbVb+/+BmDTvPAGldEkEO+NWziEE21asYJMi0uHQ==,iv:j/sex4nBa5pFOHXX6zlpJGGCxUcdxZPfHSnLA7kl0mo=,tag:BrMMLbCzwVPzFYNaClL57g==,type:str] +secret_buildoor_builder_mnemonic: ENC[AES256_GCM,data:UfA8ssN7RE7nejQMTUZ99aFr6Ty6HjG4BD7CCTRN37EL0jN/ULwE0HQEi6ydDnOWkqJmpX8u9eWl3NX1PBGyuGckHtlbtquUngIxmWy0a0Q/S0XaDVBIHyQaJTAu+RZzZEVPOxkL61mjOFSdoJKdrBPUQECdGHVYOfaUlpQR1JzCJgpi65XVTktTdoiYgfEGC3Uks3LzeMua,iv:GLzRU3KgZeHxty8u3Zxe0NV3yxu5t2Gn1tj9UzQW68o=,tag:A1N/9hoXe/4YwIX6B9Y+/Q==,type:str] +tysm_secret_access_key: ENC[AES256_GCM,data:/iXmKhlYw2bzSzirLg==,iv:MRRoJgUc6V8B8JZVGRpu6NsLkraqrf+VikEChAv3h00=,tag:HlPIdHJlSlyx7qRYxIMHsQ==,type:str] +secret_dockerhub: + username: ENC[AES256_GCM,data:CROGDtAGPOxIBOI=,iv:kd9imvumG1WRBSUKfgjfpHIOhrUy6jHL2h6808Z0IUY=,tag:SdMBMeINeT1iouCPZWBR9g==,type:str] + token: ENC[AES256_GCM,data:eSROwx/YdGm+LxygbpG1Zrqftilp5ogfXkVpratOBQId0D3a,iv:/1UyFsGqodzel2opDTR62ZVbRvQMB6EGCmehv9SlxZ8=,tag:0S/W4eXpZlVJ893u1eTpvg==,type:str] +secret_buildoor_wallet_privkey_2: ENC[AES256_GCM,data:PhlAnCwX6x/8IT7LNJMHET5lcJ5CfeHjbz6eesvqTf0A4xVo5+7DLEAoukjVDtHF2uPSlVVHaYU7ASUs+Z/IzQ==,iv:97Z90S9tUfzmGRW+QI5mvJ/Ou+13BRB2+MGGNp/EILQ=,tag:G9B84vf4iU31bvCrZlUcCQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2026-10-07T19:35:45Z" + mac: ENC[AES256_GCM,data:6RyuG02q61vn169JQAX+MVMxqaTJZe2qTj66f3V4l1cpGrtZrl9mzZXYpRiQIOpudMXfthqEVKpc8pqPYkrBRG9Upd08H2VPmq5oEdXwkpNIMPDCpHiiHsaUo+/QbHS643st6UFnDlm6681PLIrmFbmR5xE/CYJBOLmPfTe4Bjg=,iv:m1fp/HdETtBQ3hWc5YAlMpvzZiYmKvuRkBAHMH8J58Q=,tag:TFh42jOr9yUiLE9HObHCDQ==,type:str] + pgp: + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA32GcoRiZf6pAQ/+IKGAjEyWVLJbeKn+HXSWSx/ftofr+q6SawtUO7BZgUp8 + BpQuYK7/LZlsZ4Bq83/WZj0CgXDTZoVjxAoXgv1+If7tv0+tFg9l9tIuc4gAfOTn + DotvDygMRkY/kYWz+Pr+desDhma1YVxEglr/xFGmzM4v+u4Dd+ShAbwmnUsWHO0K + yucZyxIk2BeNVs0880u3pYmWwJNXckyS83kBZSP5SX+xbGpRgviGNnpk0kaYYGIr + SmAwoQi7NxVdkMd6XROV/rdL+xLjUBLPsIbVEDTqIVU6b1U7dUA6lNdKeDdqUt8B + VGpRa9AqN2HR49RnXXS56rDM2fZexVSjP6Tlc/2312zRiJzyRQhZQ/IiV84Qj0hF + KMb2nTszQr4QbhTFHD7dexnu21zQdwp1risJG9aJnBTcI/hzFiq52S8K3PDqEdlb + 50tuPoC0oG6AXXdfLycIc0VGhTSyNFr+N4VhToyRmGeP0CrUxe3XkwFiJiXxZH06 + tS/eNU5XNQ/w56sE2VVDip7OrPBwwDgiM+NaRC6lFRWKgndFmLDk9svzKzikjUCx + KrGqNQtBDUGvwTecDUXT3Ra6gZ3ibhUsyBnk1W14clm21n8ABdSOF84KgWq0rlHd + BsqIsbEqQuVdZ9i0pQO8i5YTsZU16/hV/o+U2bFlWnG0UnJDx6ubzWDX11rmvKHS + XAHTlLR/yto4S/I13bJvPG2R0hNBbvB3ZHHALnfS5lH/MuQXz6YCod7XmPcFfuKs + KJMKXusRuInXVbWWwHHKHx+GFnh/XJ/HXCx07amHyusWBWXjPqlIwKQcf1G2 + =Ko+N + -----END PGP MESSAGE----- + fp: 80DB2B4EF6CA4D9829C280605636BC0E08138A24 + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA87Wber4r4dGAQ/+JMA/0QWhmj9p4ib+o6BpC5UwBC8t3uKc+Nx3RtH1tbfu + mS4SgJNu6KR+exsH6Q3wsqxANmVKK6JjZXTiaHB2so/mT5FDctlimPbjpeqg8tcU + gec3e3jmh/QiFiIWWBy/kxORSvuQtcM8Fu5TyZiNYkV5mSmNN8YSzExjeAlOAkK1 + efBMIuoBS0QaRZqQuoqzqqnnP50FFGP9wi21FBi+ZPS2s8webYKMRoo5KySWKRh6 + PCFJ3ikJ+4n+Ln3Wq+KiWIVwzNWL9J/9bqDpIyKcA3niCvo6MGdq57AIFMV97BVs + HJ+GAKsfkZOqfASp4CEpQHZpWwc5D1T1sbq/OrPVwvKRkbcvqMM/hsqBJ9NTjR1l + 9trYk2XHr4mqYmrIsIHJCooxFyvScRsn7PA2ZSSfloLSyTZ0XaV6ueRagEN/2ny2 + 5OsSRVkCwctWWfXChj9ZGBbq+mDohobiNR09JGWrP6Xs9kHk1gvS7MgA7pbXMuad + RcKsdDFzu+Twz6nOajoBgeRl+atUmwY65e0T9wEqyCgRpBtkcK4IkblIixgVcDmb + w3GScohqQ+ehEcjTubAS5FiMs5+1B41oLtKQCooXoyfxTalwBtjaxdD+Em+nIVX7 + IiN6RpA5xcLvJyqB35LBdJinBtR24hB1S1zFJa6K7bPg8/tnfck56f3vftJZtIrS + UQEKBviA8UUfHn+oE7Gbt1ucoekkHI99smRG7rl3tWs2To6aXXD6dIHQHOwL52Qi + 3XohfZf2q4u/8a+uvwqHXNzihPDTIYNLbkjiQKrYGNsdUg== + =PTSf + -----END PGP MESSAGE----- + fp: 69F66EEA7AE36CCB77DDB8CA1BC39532FB4A2DBD + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA4hw3nPn82LyAQ//RpGs30EaSeC/rLuhGak5Yy40CZ+4R9pT/AHrizRIZxz6 + 5EodGYnRBzvX6i0Aik/M3mEHRSwIMwCfdF/tCpnjMp9exE1nu9ob32nrFtJnuRTP + E58tFRyf7Ib/1f8zMQNdId1kxtV2FY+H7fzBxrJNhiJLmpUSqzSORrPX80+ttT82 + tAxTRlk3+pYy/sgHsI4ADbGWbQUaYqxlYYbChyshT79cLlwJDqteIh83E/37Xu4s + byGdJKtzdM3PPzlhbSVvS1IGpp3xShO0/+t05Ubvk4hiJ2WKGWQjoHsNoAIsnHIR + IJX1VWAWNpiohzfuHcCPTChMHTCtINZAiCvlWfgXwMcX3ADZH0B4Gsi6rIeGb3L0 + fHk2nYdbO7i3wQrzD9qtLA0487gd8ZKh7bkxioAdzT8t7PrZpohYn6vxIYKnoRvM + FFH5RyFrGcIdxPIVQxECZcdwjWIKN1wH6KenEGe7HHAUrJH0unbr/Sbs8vEEMUFQ + vxG937vzQROVturJRdx3IE2Lzw6BrBYaGErnurVM5Po6b7jCAUwUIb82+/DYOzUY + fImO5tPWLPdSNfwdZTergdklmHPFgUt+tS0qFj9VTfLBxQmX2Dqh8WeipJxK57tD + KG7fuPk4iAVJB+oV03OQz4m/fHlMT7l3AuPnxhdkmRDIA5L4mrsqU7Zseq/OKEDS + UQEuHkX+QVDIBM/qp2UM17c9y/4W3/5FAw+R2ArZqBfjzKOTZ+LjhDNC6EDj4nmy + 1Y7BNS1dQuMBMendZrvEkDlEf7fi3o4ATCCLv6rzX4+2OQ== + =zNow + -----END PGP MESSAGE----- + fp: D1002590180DE371BCB455EAEFCD4ACD0B4D3F6E + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAw01qfIBwY0JARAAq1NTq8zuUTFoJHzkq8y1hiSod0c5Qm2p0pCijIhdHmAD + f0VUcrNbvIx6LPZWI2fhGIYwoJgQrFxQCG9a/aYYzFHhcihPJwADI1TqPZ8b2D2/ + HKLQJW2AVokvppYkLXS+KOlEEHiOmLwzbGL5fkvwAWlzEt9x3PVUH14TCcWXfuqk + 1xz4I7gXfD/Ql23pIhRzaqENpT8bXplAEMh1jEc32inCQoNbUppA+B5Xt1Ox5XVG + kl3PW16wBM0+pNN4oQjMHZfQiiJBMxr44e8ZSeYWaSjBr8jVPshp3pUq3e93JmC0 + pD4NVtnITQpIyziKyA61qKmkwC6X/g6Zn4qEcwjtvhQqs1SIH2Yu5SNOHrfpQF7P + G8UoH5gX91L3ImuvDC4wxW8eDy5bMYqi7Pxy7u+Q5fTCN2VJAvDAyGwlO76kbKZQ + wGBzGNvJNE8BM4JyJRwPq0+TmQuM4wdZvtQ0M6ddJ1rqMB6Sz3GbTg1XIAU8/q5c + rFFZNrXpZH4PkMf2kOmU5eDKWCZa3c+HrLvkgYZxo66EFcbAUHajnTqdoEX1FaD/ + cDyl1GDw8VMzbBmqA6Ad0CJ12xxDalI7us6/6fS42vO8cwhsc3Kc9KQR/qWGLfqK + I48D5KkjzzQbgb0Yh0Qt2CNu+ooSmbVxiQm1JplfnCFZIMIEhPu+L0QxnxCF09vS + UQEe1uI5Pu/LRLR/HnxD80IQ/Kz2pgqzcezmYLtQXgO3tyfeO9kRz0hciNpBsGPT + Fx21MILxKR5ypjOyspEtt7gEvhQiCy8ixc58hj+STDeH6w== + =V6Be + -----END PGP MESSAGE----- + fp: B9F81F327CF5346860E85269D7AF98F214C59E4E + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMAwDFOgk9tOQ+AQgAlRsVXPLBgH5iGDZ2RVI2Tz8vR26UoWTWtuj6ptpDR2F/ + BDMy6tohnRQiXXDHUgTm0otJaTAPNsw+90ryD1UXuGlzfYJv5tpTTTfywLEyhHtv + 5Y8T3bq7QwUFgM9e4LHVqt15b7ReJi9xkR/FoBH/4lh6NqE8Z6eD+BNszwR20aVC + u0dUwHPrROl6moPZIEvSwMPhCGbI5ZO2S6g+iGCsXNOLtCNtSVQw9Czf8JKMlCpD + EiyQOOwTjZZ6sGj7wDvxZdyW0K5LH7fsHms8V5+Pzpk+gLs9QKhYL3xa4KJg5Tg9 + 2rdqdfNQcECivtQH2wgVwPikQe5BzUx31cYwAvp/P9JcAbvqgXUG30AUrwojSi7d + RhfRVACpmUUoc+FsBjdUDux6GXkdkoIOxsJ3kPg37nJiwCzq3uXJptwoykjy+Por + rMTp5fr7JCMMAzC+mLyrpvr3Z77EffAEgULbTck= + =xfyc + -----END PGP MESSAGE----- + fp: 0600D41E1313E31016F7C55BF597BC0C5F22D1A2 + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAz4a8AV36xppAQ//THNYUTc2CaY317l+48yibj89XMdC3O9S3XWIi0yQ88DT + MEYUA8j74oJwIBRX9H44nTpL0Jfbg4KqhdPSaS/ZXtgcxTrs3aym008C1Auf974D + OGWEG29ZavXiLj9N+Qy4k3jTGhXjyjp8qpis/wpR/yhLGRrFjorkNkJ0xf6bHBVg + DNpQSGfyY5cBSLCOSb51pZXJ1PBmWwKoIhtpZuFeV9OoayN42kDNTQdC1v+PICYK + PqHLcRB5e6mayPDmZu7YAzFui4yeTSpR/FDP54ckQuIZxN+Pw66n8ocRme/T/Mhz + H2o7RNwtGxgDRYyAz2puag+WFFZoo7tfbSXykSFEFBBQrP6OoQ3ZS5BuxuGSrwNS + GQ/AnzPHL4FfTBMz8tEfeOfwBWTiiZ0IVvEKa0YmGk7d3mlMGul08o5nTfgxI3W2 + YNp7E5TTEgpCfRHBUtH3hMIRwU0ecjTuM91/9kF8Up1UlDRABJ7DsQXZpEcH+SYG + XTLrhl0aHgrxSSmFv3I6JuiSuYQBpfmug6Ed9GJWrwMNFlr1QkblNIF1WEVbaQ6k + j37XL5VOe70tTM0FL3rqYCCuDKoJ2eUHGvJ2cjPekDxhQ/4ps8HwevG2v/oYY7VY + 7eSkC8QWs6JlWPZZJet5hmhRP2BF7LflqaPyAfyrR85kLH/ldS87A6kcIP2QDB/S + XAHKuYj9EjVjEBDjTIz0K2+ctHUR9TikwGan7lq5zFDdxVw6Qs46pGwP2FuXslSs + XK8WKviNRZ99UHHYpKAf3j8ztsKBaVlrM0f/ZUXsmIvPQIaHfLcqK/gmW76O + =Wzuw + -----END PGP MESSAGE----- + fp: 29C50D01122FDE78E257482DAA497EB2610A8435 + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAxYzhHRfYJYtAQ//QHZYI5KPXJIeodQqPFJl8L/LnPa6ycOoNXuBngsYNBw3 + mT12bU/+zhiQd39Sh2xRhGGh3sK0SCCfarH/rjMhG/G+7hobFITh+jYMNuJO9kNd + SBii1HyvuwOkqSIYdVToiAKvmAXHQqAjNNCaHT2FBKZQSxq7eMC9/7GBEVBxiYlC + Xw9LoAAf6vrSJq/7nxokkAFYBwjH6H06lv0cOiCNuci3oKzE64toAo4zSwcj9yzQ + hFI4Q+Er62Ep8fMRxsy9NkxZNgLruPzssGxa/n4jk9t7fF9EQmyoeRVLUxNvr+w3 + W7sol2sxTupj2Bxtz9SH4h3Ubuey4HwyM5mD8n/euU+K1pO5aPBGf2oz1fbK3D0K + 9czZpSS/ehWjeytpjJCAmL6m2rRtd1IM29Bt9RNnRaL2hqUwliNVXfo2TKsilp0i + WL0f0ZpDElc9ttxWOFBir8fCCMFw6Oe/zoicEkxC7/W4iPyLTyHFZAlio2cBiV0I + 9dTtUQbKW1Yav30Y4Wg2SqlOEUXki5KkZ+GyWNRwVLePh3Ec0MbcXQRREesK0goq + pUV42dpAV52b5svPndZa8AJZe3RFbqIcFb08UNsDwFSN/ZGLH8geu+X7ROdq8XxN + yclqoVUv7Aq65kgaSqFszSYEG2+hoNVYpdYI/OTYFa7r7922fhqQSvPPUhm3JtDS + XAHE3c57G66vrGGwnO5J7Sy6DVS3TpFNONq0untKm9lj6M6QiME4P/tT9UIq5Uca + S0mO0LwZPogjfBgE1bFcUfk6I6Awo/6QupkKyqC7PWgsXnK0E1iJyN79Sukt + =cfzI + -----END PGP MESSAGE----- + fp: 9BE537027CB7467923E240FF2AED09371C121F91 + - created_at: "2025-10-27T13:25:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DjS7VXZx9i8YSAQdAkcV1Mg7BYz1ISw2x8guADkVnYfZbcY1dxqt4M2QidkIw + Uws9+CDwZfSJSddGvcFkFTSUMKqjW2zY0zL12i+H8ixluWM6rJ2U49OnjAd7chip + 1GYBCQIQkV5CNAErwKJ7hD7y1Io4TGvsuRkYsm0L4CwR/9sy/T63eKHMqerVsRoN + 2FtEFaTV2Xc1iPq78GEJmBSsR8sZI1+EnAlZ0HRz1FUYsuL3/zRQnu2fucKkJBIs + axCTPk8EX80= + =8dXI + -----END PGP MESSAGE----- + fp: F93098C3D8ED15D0924A6DDAEB4E93A88660C55B + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/ansible/inventories/msf-2/group_vars/all/all.yaml b/ansible/inventories/msf-2/group_vars/all/all.yaml new file mode 100644 index 00000000..abb44b7a --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/all/all.yaml @@ -0,0 +1,598 @@ +--- +# ░██████╗░██╗░░░░░░█████╗░██████╗░░█████╗░██╗░░░░░  ██╗░░░██╗░█████╗░██████╗░░██████╗ +# ██╔════╝░██║░░░░░██╔══██╗██╔══██╗██╔══██╗██║░░░░░  ██║░░░██║██╔══██╗██╔══██╗██╔════╝ +# ██║░░██╗░██║░░░░░██║░░██║██████╦╝███████║██║░░░░░  ╚██╗░██╔╝███████║██████╔╝╚█████╗░ +# ██║░░╚██╗██║░░░░░██║░░██║██╔══██╗██╔══██║██║░░░░░  ░╚████╔╝░██╔══██║██╔══██╗░╚═══██╗ +# ╚██████╔╝███████╗╚█████╔╝██████╦╝██║░░██║███████╗  ░░╚██╔╝░░██║░░██║██║░░██║██████╔╝ +# ░╚═════╝░╚══════╝░╚════╝░╚═════╝░╚═╝░░╚═╝╚══════╝  ░░░╚═╝░░░╚═╝░░╚═╝╚═╝░░╚═╝╚═════╝░ + +domain: ethpandaops.io +network_subdomain: "{{ ethereum_network_name }}.{{ domain }}" +network_server_subdomain: srv.{{ network_subdomain }} +server_fqdn: "{{ inventory_hostname }}.{{ network_server_subdomain }}" +network_iteration: "{{ ethereum_network_name.split('-')[1:] | join('-') }}" + +ethereum_network_id: >- + {{ (lookup('file', eth_testnet_config_local_dir_src + '/genesis.json') | from_json).config.chainId }} +ethereum_network_deposit_contract: >- + {{ lookup('file', eth_testnet_config_local_dir_src + '/deposit_contract.txt') }} +ethereum_network_deposit_contract_block: >- + {{ lookup('file', eth_testnet_config_local_dir_src + '/deposit_contract_block.txt') }} + +ethereum_node_rpc_prefix: rpc- # prefix for rpc URLs +ethereum_node_beacon_prefix: bn- # prefix for beacon URLs +ethereum_node_rcp_hostname: "{{ ethereum_node_rpc_prefix }}{{ server_fqdn }}" +ethereum_node_beacon_hostname: "{{ ethereum_node_beacon_prefix }}{{ server_fqdn }}" + +primary_bootnode: bootnode-1 + +### Checkpoint sync specific for Ansible & Kubernetes +checkpoint_sync_node: "{{ primary_bootnode }}" +# Shadowfork: every CL starts from the generated genesis.ssz (there is no older +# finalized state to sync from), so checkpoint sync stays off. +ethereum_node_cl_checkpoint_sync_enabled: false + +checkpoint_sync_url: https://checkpoint-sync.{{ ethereum_network_name }}.{{ domain }} +# checkpoint_sync_url: https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}{{ checkpoint_sync_node }}.{{ ethereum_network_name }}.{{ domain }} +checkpoint_sync_url_kube: https://@{{ + ethereum_node_beacon_prefix }}{{ checkpoint_sync_node }}.{{ ethereum_network_name }}.{{ domain }} + +# ██████╗░░█████╗░██╗░░░░░███████╗  ██╗░░░██╗░█████╗░██████╗░░██████╗ +# ██╔══██╗██╔══██╗██║░░░░░██╔════╝  ██║░░░██║██╔══██╗██╔══██╗██╔════╝ +# ██████╔╝██║░░██║██║░░░░░█████╗░░  ╚██╗░██╔╝███████║██████╔╝╚█████╗░ +# ██╔══██╗██║░░██║██║░░░░░██╔══╝░░  ░╚████╔╝░██╔══██║██╔══██╗░╚═══██╗ +# ██║░░██║╚█████╔╝███████╗███████╗  ░░╚██╔╝░░██║░░██║██║░░██║██████╔╝ +# ╚═╝░░╚═╝░╚════╝░╚══════╝╚══════╝  ░░░╚═╝░░░╚═╝░░╚═╝╚═╝░░╚═╝╚═════╝░ + +# role: ethpandaops.general.ethereum_genesis +ethereum_genesis_generator_container_image: "{{ default_tooling_images.ethereum_genesis_generator }}" +ethereum_genesis_generator_version: "{{ ethereum_genesis_generator_container_image.split(':')[-1] }}" +ethereum_genesis_network_seed: glamsterdam-msf-2 +# Chain id 1 = egg mainnet shadowfork mode: keeps osaka/bpo1/bpo2 at their real times, +# adds our amsterdamTime, no allocs. +ethereum_genesis_chain_id: "1" +ethereum_genesis_fork_version_suffix: "{{ 999999 | random(start=100000, seed=ethereum_genesis_network_seed) }}" +ethereum_genesis_generator_output_dir: ../network-configs/{{ network_iteration }} +# 2026-10-07 19:00 UTC +ethereum_genesis_timestamp: 1791399600 +ethereum_genesis_timedelay: 0 +# jochemnet (benchmarkoor): mainnet 24350000 + 1 s bloat blocks up to 24402727, not on +# mainnet. Gate before genesis: eth_config shows our amsterdamTime on every EL. +# shadowfork_height is the snapshot; the head is 2 blocks later, built on a restored geth and +# imported into every EL over the engine API (scripts/shadowfork-prestate/): +# 24402728: benchmarkoor's funding (15001 withdrawals of 2^64-1 gwei, funding.txt of +# gas-benchmarks amsterdam-repricings-v5.2.0) + 1 ETH for a throwaway deployer +# 24402729: the two EIP-8282 contracts via the 0x4e59 CREATE2 factory +shadowfork_height: 24402727 +shadowfork_head_height: 24402729 +gloas_fork_epoch: 40 +ethereum_genesis_mnemonic: "{{ secret_genesis_mnemonic }}" +ethereum_genesis_generator_config_files: + cl/config.yaml: "{{ lookup('ansible.builtin.url', 'https://raw.githubusercontent.com/ethpandaops/ethereum-genesis-generator/v{{ethereum_genesis_generator_version}}/config-example/cl/config.yaml', + split_lines=false) }}" # noqa yaml[line-length] + cl/mnemonics.yaml: "{{ lookup('ansible.builtin.url', 'https://raw.githubusercontent.com/ethpandaops/ethereum-genesis-generator/v{{ethereum_genesis_generator_version}}/config-example/cl/mnemonics.yaml', + split_lines=false) }}" # noqa yaml[line-length] + el/genesis-config.yaml: "{{ lookup('ansible.builtin.url', 'https://raw.githubusercontent.com/ethpandaops/ethereum-genesis-generator/v{{ethereum_genesis_generator_version}}/config-example/el/genesis-config.yaml', + split_lines=false) }}" # noqa yaml[line-length] + # Network identity and fork schedule only - every spec parameter not listed + # here follows the egg defaults, which match sepolia's Gloas config.yaml. + # Shadowfork specifics: + # - SHADOW_FORK_FILE: eth_getBlockByNumber("0x1745b29", true) after the prestate import (full + # txs; jochemnet publishes none, the one in the erigon tarball is block 24350000). + # - DEPOSIT_CONTRACT_ADDRESS must be mainnet's or the generator refuses. + # - GAS_LIMIT_SCHEDULE: 200M at the Gloas epoch, like sepolia. + # - WITHDRAWAL_TYPE=0x01 + VALIDATOR_BALANCE=10000 ETH (every fusaka shadowfork + # did this): shadowfork mode adds no premines, so the 9968 ETH excess per + # validator is swept to WITHDRAWAL_ADDRESS (mnemonic index 0, + # 0x454b0EA7d8aD3C56D0CF2e44Ed97b2Feab4D7AF2) and funds the tooling from there. + # Index 0 is empty on real mainnet. + # Not set because shadowfork mode ignores them: EL premines, GENESIS_GASLIMIT, + # DEPLOY_EIP8282_CONTRACTS (deployed in block 24402729, see shadowfork_head_height). + values.env: |- + export CHAIN_ID="{{ ethereum_genesis_chain_id }}" + export EL_AND_CL_MNEMONIC="{{ ethereum_genesis_mnemonic }}" + export NUMBER_OF_VALIDATORS=36000 + export GENESIS_FORK_VERSION="0x10{{ ethereum_genesis_fork_version_suffix }}" + export ALTAIR_FORK_VERSION="0x20{{ ethereum_genesis_fork_version_suffix }}" + export BELLATRIX_FORK_VERSION="0x30{{ ethereum_genesis_fork_version_suffix }}" + export CAPELLA_FORK_VERSION="0x40{{ ethereum_genesis_fork_version_suffix }}" + export DENEB_FORK_VERSION="0x50{{ ethereum_genesis_fork_version_suffix }}" + export ELECTRA_FORK_VERSION="0x60{{ ethereum_genesis_fork_version_suffix }}" + export FULU_FORK_VERSION="0x70{{ ethereum_genesis_fork_version_suffix }}" + export GLOAS_FORK_VERSION="0x80{{ ethereum_genesis_fork_version_suffix }}" + export GLOAS_FORK_EPOCH="{{ gloas_fork_epoch }}" + export GAS_LIMIT_SCHEDULE='[{"epoch": {{ gloas_fork_epoch }}, "gas_limit": 200000000}]' + export GENESIS_TIMESTAMP={{ ethereum_genesis_timestamp }} + export GENESIS_DELAY={{ ethereum_genesis_timedelay }} + export WITHDRAWAL_TYPE=0x01 + export VALIDATOR_BALANCE=10000000000000 + export DEPOSIT_CONTRACT_ADDRESS=0x00000000219ab540356cBB839Cbe05303d7705Fa + export SHADOW_FORK_FILE=/config/shadowfork_block.json + # Block 24402729 from the bootnode geth (eth_getBlockByNumber, full txs). + shadowfork_block.json: "{{ lookup('ansible.builtin.file', (ansible_inventory_sources[0] | dirname) ~ '/files/shadowfork_block.json') }}" + +# role: ethpandaops.general.snapshot_fetcher (variable names reused by +# ansible/shadowfork.yaml, which downloads with parallel ranged GETs instead). +# snapshot_fetcher_out_dir/_filename are set per EL in its group_vars file. +snapshot_fetcher_network: jochemnet +snapshot_fetcher_client: "{{ ethereum_node_el }}" +snapshot_fetcher_block: "{{ shadowfork_height }}" + +ethereum_genesis_validator_keys_output_dir: "{{ ansible_inventory_sources[0] | dirname }}/files/validator_keys" +ethereum_genesis_validator_bls_change_execution_address: "{{ ethereum_node_cl_validator_fee_recipient }}" +ethereum_genesis_validator_keyranges: >- + {%- set ns = namespace() -%} + {%- set ns.ethereum_genesis_validator_keyranges = {} -%} + {%- for host in groups['all'] -%} + {%- if hostvars[host].validator_start is defined and hostvars[host].validator_end is defined -%} + {%- set v = {'start':hostvars[host].validator_start, 'end': hostvars[host].validator_end } -%} + {%- set _ = ns.ethereum_genesis_validator_keyranges.update({host: v}) -%} + {%- endif -%} + {%- endfor -%} + {{ ns.ethereum_genesis_validator_keyranges }} + +# Validator display names for the /meta/api/v1/validator-ranges.json endpoint +# (consumed by dora, assertoor and beacon-metrics-gazer). A host may set +# validator_display_name to be shown instead of its inventory hostname (used for +# experimental-build hosts). Only the displayed segment name changes: +# key generation and syncing still key by hostname through +# ethereum_genesis_validator_keyranges above. +eth_inventory_web_segments_json: >- + {%- set segs = [] -%} + {%- for host, kr in (ethereum_genesis_validator_keyranges | default({})).items() -%} + {%- set _ = segs.append({'name': hostvars[host].validator_display_name | default(host), 'start': kr.start | int, 'end': (kr.end | int) - 1}) -%} + {%- endfor -%} + {{ segs | to_json }} + +# role: ethpandaops.general.bootstrap +bootstrap_default_user_authorized_keys_plain: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbboxOo0jyL3DNxqZ6UTEnZPEzPDPnujEYaClqNWSLWkphczHKAnJPkrwbAWB4JbJKjsAJ5kn53f10KPnUyZvJ5Jn8Rpf7RM7+56MYaBg84gVoA2KeIYxUa7h8neY7J61Galp0c6cOK+hp1lPsoiBSdCW/Rtbv6ALCcVe+4+uCW5FRoJcNRJfGRLRnjh1pw57HQw9O55mf319s4rVUq4umznQ0CciEx3rVMtXf4xjIZDZAhNpGaBh8AtHauaMZCOGociAIquYYqoSQnnmnOBiduRa5OkvGZomgybNQivlYboDeF6sQ71KVzRXSI+mxCYbSp246lqSdQtQsjFA54NYl/qWgAql0uqCqsZidW+XBjquyItRl2Rfzzy5Fk/gMOAJXHQYp4POfgFbqtxjWpfnuOKqW/1IGWcIt2g016effUqGgj/oePX0g+duFdszSKK773rJBySgafFF6XWNqagrLmE4LUGC+6P3oxzYTSFGeUVA21OayL+K40XPpJti5zns= + # devops-eth2-shared" + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWh9NW66VD4BPKETNyZeZrGN1f7G6dkihW3eAc7cbJPFQGIpnWc2tGq5o13vWW+SoCh16nkYM2oak+PJQxXYTiQnrMJSmSFd7E0DmdcoKadGJEnfosrH++aOZf/eVLe5q3E9NQFVSdOPo1MCRRTuZxPkuMxS6QikW3otWrA3F2vFgmYyki3Cy8huQzHKUZGicividYcUSFTydR2L0oWUNve3FyqMQQQPnfaJ1RvrkeGtdhRSAxa6L0jzgRK7fjpUyhKOofr7kCKARGELRRiB9QikRAoHU2/D/2jtJjKlTCJxArzXyDF2IcQCco+5Oe9x4c7Xch32dbscJSmjaAvsxRnu7GEFCS7b6kKGvwcoq5vJzvp3RBBR7Mosxv6pcM/q7Z4RhXOFVFFiPVl1dqkqSPkUrHwg8LtWOxC+GAl36vxhHLdDEV/RhbSAzO6SfYEWYGH1w7u4oiy2XAT2cNCO0j0tSHS5chX+d7TzwAbBE2HuPL84GVGHZG875hmiE+Dok= + # github-actions-ci" +# Per-client team SSH rosters are fetched at runtime from the coredevs registry +# (see each client group_vars file). https://github.com/ethpandaops/coredevs +coredevs_api_url: https://coredevs.analytics.production.platform.ethpandaops.io + +# ethpandaops core team, sourced from the coredevs ethpandaops team. Handles not +# in the coredevs team go in the extra list. +github_all_extra_users: + - bharath-123 +bootstrap_default_user_authorized_keys_github_all: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/ethpandaops?format=txt') | select | list) + + github_all_extra_users }} + +bootstrap_default_user_authorized_keys_github: > + {{ + (bootstrap_default_user_authorized_keys_github_all | default([])) + + (bootstrap_default_user_authorized_keys_github_team_el | default([])) + + (bootstrap_default_user_authorized_keys_github_team_cl | default([])) + }} + +node_exporter_container_image: "{{ default_tooling_images.node_exporter }}" +prometheus_container_image: "{{ default_tooling_images.prometheus }}" +vector_container_image: "{{ default_tooling_images.vector }}" +json_rpc_snooper_container_image: "{{ default_tooling_images.json_rpc_snooper }}" + +# role: ethpandaops.general.ethereum_node +ethereum_node_images_always_pull: true +ethereum_node_metrics_exporter_enabled: true +ethereum_node_xatu_sentry_enabled: true +ethereum_node_cl_validator_enabled: "{{ validator_start is defined and validator_end is defined }}" +ethereum_node_cl_validator_fee_recipient: "0xf97e180c050e5Ab072211Ad2C213Eb5AEE4DF134" +ethereum_node_cl_ports_p2p_tcp: 9000 +ethereum_node_cl_ports_p2p_udp: 9000 +ethereum_node_cl_ports_http_beacon: 5052 +ethereum_node_cl_ports_metrics: 5054 +ethereum_node_el_ports_p2p_tcp: 30303 +ethereum_node_el_ports_p2p_udp: 30303 +ethereum_node_el_ports_http_rpc: 8545 +ethereum_node_el_ports_ws_rpc: 8546 +ethereum_node_el_ports_engine: 8551 +ethereum_node_el_ports_metrics: 6060 +ethereum_node_ipv6_enabled: true +ethereum_node_el_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.execution) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +ethereum_node_cl_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.beacon) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +ethereum_node_validator_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.validator) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +ethereum_node_snooper_rpc_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.snooper_rpc) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +ethereum_node_snooper_engine_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.snooper_engine) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +ethereum_node_skip_cleanup: true +# Watchtower: daily image check staggered per host across a 3-hour window +# ending at 18:00 CET/CEST (seeded by network+hostname) so fleet-wide image +# rolls restart nodes one by one instead of all at once (devnet-9 lesson). +# Off for the shadowfork: its window (13:00-16:00 UTC) would roll clients around +# genesis and the fork; images are pinned for the run, rolls are on request. +ethereum_node_docker_watchtower_enabled: false +ethereum_node_docker_watchtower_containers_list: + - execution + - beacon + - validator + - xatu-sentry +docker_watchtower_container_additional_args: + - "--schedule=0 {{ 60 | random(seed=ethereum_network_name ~ inventory_hostname ~ 'min') }} {{ 15 + (3 | random(seed=ethereum_network_name ~ inventory_hostname ~ 'hour')) }} * * *" + - --stop-timeout=300s + - --include-restarting +docker_watchtower_container_env: + TZ: Europe/Paris +ethereum_node_json_rpc_snooper_engine_enabled: true +ethereum_node_json_rpc_snooper_engine_name: snooper-engine +ethereum_node_json_rpc_snooper_engine_port: 8561 +ethereum_node_json_rpc_snooper_engine_public_port: 8961 +ethereum_node_json_rpc_snooper_engine_container_env: + SNOOPER_API_PORT: "{{ ethereum_node_json_rpc_snooper_engine_public_port | quote }}" + SNOOPER_API_AUTH: "{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}" + SNOOPER_XATU_ENABLED: "true" + SNOOPER_XATU_NAME: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + SNOOPER_XATU_NETWORK_NAME: "{{ ethereum_network_name }}" + SNOOPER_XATU_NETWORK_ID: "{{ ethereum_network_id }}" + SNOOPER_XATU_OUTPUTS: "xatu:{{ xatu_sentry_config_server_address }}" + SNOOPER_XATU_TLS: "true" + SNOOPER_XATU_HEADERS: "authorization=Basic {{ (xatu_sentry_config_server_auth_user + ':' + xatu_sentry_config_server_auth_password) | b64encode }}" + SNOOPER_JWT_SECRET: /execution-auth.jwt +json_rpc_snooper_container_volumes: + - /data/execution-auth.secret:/execution-auth.jwt:ro + +# role: ethpandaops.general.generate_kubernetes_config +# Nickname for this devnet, used for faucet titles and dora/forkmon labels. +gen_kubernetes_config_network_display_name: glamsterdam-msf-2 +gen_kubernetes_config_dora_execution_snooper_port: 8961 +gen_kubernetes_config_dora_frontend_rainbowkit_id: 15fe4ab4d5c0bcb6f0dc7c398301ff0e +gen_kubernetes_config_dora_execution_endpoints_url: https://raw.githubusercontent.com/ethpandaops/{{ devnet_name }}-devnets/refs/heads/master/kubernetes/{{ + network_iteration }}/dora/endpoints-el.yaml +gen_kubernetes_config_dora_consensus_endpoints_url: https://raw.githubusercontent.com/ethpandaops/{{ devnet_name }}-devnets/refs/heads/master/kubernetes/{{ + network_iteration }}/dora/endpoints-cl.yaml +gen_kubernetes_config_dora_api_secret: "{{ secret_dora_api_key }}" +gen_kubernetes_config_dora_xatu_enabled: true +# Don't let tooling call debug/trace endpoints: disable dora execution indexer +# trace fetching (debug_traceBlockBy*) and the tracoor chart entirely. +gen_kubernetes_config_dora_execution_indexer_traces: false +gen_kubernetes_config_disabled_services: + - tracoor + # faucet_private_key (mnemonic-4) is 7702-delegated to a sweeper in mainnet state; + # faucet-agents uses its own key. + - faucet + # Its "add to MetaMask" offers chain id 1: a tx signed there with a real key is valid on mainnet. + - homepage +# The EIP-8282 contracts are in block 24402729, so this only checks them (it skips the deploy +# when the deposit contract has code). +gen_kubernetes_config_assertoor_tests: + - id: deploy-eip8282-contracts + file: https://raw.githubusercontent.com/ethpandaops/assertoor/master/playbooks/gloas-dev/deploy-eip8282-contracts.yaml + schedule: + startup: true + +# role: ethpandaops.general.xatu_sentry +xatu_sentry_container_image: "{{ default_tooling_images.xatu_sentry }}" +xatu_sentry_config_name: "{{ ethereum_network_name }}-{{ inventory_hostname }}" +# Per-devnet xatu pipeline (mca-xatu-devnets in platform): the ingest host is +# derived from the network name so new devnets need no wiring here. +xatu_sentry_config_server_address: "{{ ethereum_network_name }}-ingest.xatu.ethpandaops.io:443" +xatu_sentry_config_server_auth_user: "{{ secret_xatu_sentry.user }}" +xatu_sentry_config_server_auth_password: "{{ secret_xatu_sentry.password }}" +xatu_sentry_config_network_name_override: "{{ ethereum_network_name }}" + +# Full config override: adds the Gloas SSE topics, which the sentry's default +# subscription set deliberately excludes. +xatu_sentry_config: | + logging: "info" + metricsAddr: ":9090" + name: "{{ xatu_sentry_config_name }}" + ntpServer: time.google.com + ethereum: + beaconNodeAddress: {{ xatu_sentry_config_beacon_uri }} + overrideNetworkName: {{ xatu_sentry_config_network_name_override }} + beaconSubscriptions: + - attestation + - single_attestation + - block + - block_gossip + - chain_reorg + - finalized_checkpoint + - head + - head_v2 + - voluntary_exit + - contribution_and_proof + - blob_sidecar + - data_column_sidecar + - execution_payload + - execution_payload_gossip + - execution_payload_available + - execution_payload_bid + - payload_attestation_message + - proposer_preferences + outputs: + - name: grpc + type: xatu + config: + address: {{ xatu_sentry_config_server_address }} + tls: {{ xatu_sentry_config_server_tls_enabled }} + maxQueueSize: 100000 + workers: 5 + headers: + authorization: "Basic {{ (xatu_sentry_config_server_auth_user + ":" + xatu_sentry_config_server_auth_password) | b64encode }}" + +# role: ethpandaops.general.ethereum_metrics_exporter +ethereum_metrics_exporter_container_image: "{{ default_tooling_images.ethereum_metrics_exporter }}" +ethereum_metrics_exporter_container_user: root +ethereum_metrics_exporter_container_volumes: + - "{{ ethereum_metrics_exporter_dir_config }}:/config:ro" + - /var/run/docker.sock:/var/run/docker.sock:ro + - /data:/data:ro +ethereum_metrics_exporter_config: | + consensus: + enabled: true + url: "{{ ethereum_metrics_exporter_cl_endpoint }}" + name: "consensus-client" + execution: + enabled: true + url: "{{ ethereum_metrics_exporter_el_endpoint }}" + name: "execution-client" + modules: + - "eth" + - "net" + - "web3" + - "txpool" + docker: + enabled: true + endpoint: "unix:///var/run/docker.sock" + interval: "10s" + containers: + - name: "execution" + type: "execution" + filesystem: + enabled: true + volumes: + - name: "*" + monitor: true + - path: "/execution-auth.jwt" + monitor: false + - path: "/network-config" + monitor: false + - name: "beacon" + type: "consensus" + filesystem: + enabled: true + volumes: + - name: "*" + monitor: true + - path: "/execution-auth.jwt" + monitor: false + - path: "/network-config" + monitor: false + +# role: eth_testnet_config +eth_testnet_config_dir: /data/ethereum-network-config/metadata +eth_testnet_config_local_dir_enabled: true +eth_testnet_config_local_dir_src: "{{ ethereum_genesis_generator_output_dir }}/metadata/" + +# role: gen_basic_auth_nginx +gen_basic_auth_nginx_name: "{{ secret_nginx_shared_basic_auth.name }}" +gen_basic_auth_nginx_password: "{{ secret_nginx_shared_basic_auth.password }}" +docker_nginx_proxy_container_image: "{{ default_tooling_images.nginx_proxy }}" +docker_nginx_proxy_docker_gen_container_image: "{{ default_tooling_images.nginx_proxy_gen }}" +docker_nginx_proxy_acme_companion_container_image: "{{ default_tooling_images.nginx_proxy_acme }}" + +# role: ethpandaops.general.docker_nginx_proxy +docker_nginx_proxy_public_ipv6: "{{ host_ipv6_prefix | ansible.utils.ipaddr(service_ipv6_offsets.nginx_proxy) | ansible.utils.ipaddr('address') if (docker_network_enable_ipv6 + | bool) else '' }}" +docker_nginx_proxy_container_name: nginx-proxy +docker_nginx_proxy_default_email: certs@{{ domain }} +docker_nginx_proxy_docker_gen_container_name: nginx-proxy-gen +docker_nginx_proxy_docker_gen_container_env: + RESOLVERS: 1.1.1.1 + ENABLE_IPV6: "true" +docker_nginx_proxy_acme_companion_enabled: false +docker_nginx_proxy_acme_companion_container_name: nginx-proxy-acme +docker_nginx_proxy_acme_companion_container_env: + DEFAULT_EMAIL: "{{ docker_nginx_proxy_default_email }}" + NGINX_PROXY_CONTAINER: "{{ docker_nginx_proxy_container_name }}" + NGINX_DOCKER_GEN_CONTAINER: "{{ docker_nginx_proxy_docker_gen_container_name }}" + ACME_CA_URI: https://acme.zerossl.com/v2/DV90 + ACME_EAB_KID: "{{ secret_zerossl.ACME_EAB_KID }}" + ACME_EAB_HMAC_KEY: "{{ secret_zerossl.ACME_EAB_HMAC_KEY }}" +docker_nginx_proxy_container_networks: >- + {{ [{'name': 'shared'} | combine({'ipv6_address': docker_nginx_proxy_public_ipv6} if (docker_nginx_proxy_public_ipv6 | default('') | length > 0) else {})] }} +docker_nginx_proxy_docker_gen_container_networks: [] +docker_nginx_proxy_acme_companion_container_networks: "{{ docker_networks_shared }}" +docker_nginx_proxy_acme_monitor_container_networks: "{{ docker_networks_shared }}" +docker_nginx_proxy_acme_monitor_enabled: false +docker_nginx_proxy_datadir: /opt/nginx-proxy +docker_nginx_proxy_container_volumes: + - "{{ docker_nginx_proxy_datadir }}/conf:/etc/nginx/conf.d" + - "{{ docker_nginx_proxy_datadir }}/vhost:/etc/nginx/vhost.d" + - "{{ docker_nginx_proxy_datadir }}/certs:/etc/nginx/certs:ro" + - html:/usr/share/nginx/html + - "{{ docker_nginx_proxy_datadir }}/htpasswd:/etc/nginx/htpasswd:ro" +docker_nginx_proxy_cert_loader_container_image: "{{ default_tooling_images.nginx_proxy_cert_loader }}" +docker_nginx_proxy_cert_linker_container_image: "{{ default_tooling_images.nginx_proxy_cert_linker }}" +docker_nginx_proxy_cert_loader_container_networks: [] +docker_nginx_proxy_cert_linker_container_networks: [] +docker_nginx_proxy_wildcard_cert: "{{ network_server_subdomain }}" +docker_nginx_proxy_wildcard_cert_url: http://cert.{{ network_server_subdomain }}/{{ network_server_subdomain }}-latest.tar.enc +docker_nginx_proxy_wildcard_cert_psk: "{{ secret_cert_encryption_psk }}" + +# OTLP egress to the prod analytics gateway (Vector ships container logs here). +# Ingress identity (auth username + ingress_user tag) is the current devnet name +# rather than secret_loki.username: the sops username drifts stale between devnet +# iterations, while the gateway only validates the (constant) password — so using +# the network name keeps log attribution correct no matter what sops holds. +otlp_endpoint: https://otlp.analytics.production.platform.ethpandaops.io +otlp_deployment_env: production + +# role: ethpandaops.general.vector +vector_config: | + # Docker container logs (clean per-container metadata straight from the Docker API) + [sources.in] + type = "docker_logs" + exclude_containers = [ + "{{ vector_container_name }}", + "ethereum-metrics-exporter", + "nginx-proxy", + "node_exporter", + "prometheus", + "snooper-", + ] + + # Shape docker_logs events into an OTLP resourceLogs envelope with full metadata. + [transforms.otel_shape] + type = "remap" + inputs = ["in"] + source = ''' + ts_ns = to_string(to_unix_timestamp(now(), unit: "nanoseconds")) + if is_timestamp(.timestamp) { + ts_ns = to_string(to_unix_timestamp!(.timestamp, unit: "nanoseconds")) + } + msg = string(.message) ?? "" + ctr = string(.container_name) ?? "unknown" + img = string(.image) ?? "" + strm = string(.stream) ?? "stdout" + + # Extract the source log level: JSON-structured lines (.level / .severity) + # first, then logfmt (level=xxx), then a level token near the start of a + # plain-text line. Client formats vary (lighthouse truncates to 4 chars: + # DEBG/ERRO/CRIT), so the text matcher allows a union of forms. SeverityText + # keeps the source's exact text; only SeverityNumber is normalised to the + # OTel scale (https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber). + # Unrecognised lines are left unset rather than guessed. + sevtext = "" + if starts_with(msg, "{") { + j = parse_json(msg) ?? {} + if is_string(j.level) { + sevtext = string!(j.level) + } else if is_string(j.severity) { + sevtext = string!(j.severity) + } + } + if sevtext == "" { + lm = parse_regex(msg, r'(?i)\b(?:level|lvl|severity)="?(?P[a-z]+)') ?? {} + if is_string(lm.l) { + sevtext = string!(lm.l) + } else { + hm = parse_regex(truncate(msg, 48), r'(?i)\b(?PTRACE|DEBUG|DEBG|DBUG|NOTICE|INFO|WARNING|WARN|ERROR|ERRO|CRITICAL|CRIT|FATAL|PANIC)\b') ?? {} + if is_string(hm.l) { + sevtext = string!(hm.l) + } + } + } + lvl = upcase(sevtext) + sevnum = 0 + if lvl == "TRACE" { + sevnum = 1 + } else if lvl == "DEBUG" || lvl == "DEBG" || lvl == "DBUG" { + sevnum = 5 + } else if lvl == "INFO" || lvl == "INFOR" || lvl == "NOTICE" { + sevnum = 9 + } else if lvl == "WARN" || lvl == "WARNING" { + sevnum = 13 + } else if lvl == "ERROR" || lvl == "ERRO" { + sevnum = 17 + } else if lvl == "CRIT" || lvl == "CRITICAL" { + sevnum = 18 + } else if lvl == "FATAL" || lvl == "PANIC" { + sevnum = 21 + } + + attrs = [ + {"key": "service.name", "value": {"stringValue": ctr}}, + {"key": "container.name", "value": {"stringValue": ctr}}, + {"key": "container.image.name", "value": {"stringValue": img}}, + {"key": "deployment.environment", "value": {"stringValue": "{{ otlp_deployment_env }}"}}, + {"key": "forwarder", "value": {"stringValue": "vector"}}, + {"key": "ingress_user", "value": {"stringValue": "{{ ethereum_network_name }}"}}, + {"key": "network", "value": {"stringValue": "{{ ethereum_network_name }}"}}, + {"key": "testnet", "value": {"stringValue": "{{ ethereum_network_name }}"}}, + {"key": "instance", "value": {"stringValue": "{{ inventory_hostname }}"}}, + {"key": "host.name", "value": {"stringValue": "{{ inventory_hostname }}"}} + ] + {%- if ethereum_node_cl is defined +%} + attrs = push(attrs, {"key": "ethereum_cl", "value": {"stringValue": "{{ ethereum_node_cl }}"}}) + {%- endif +%} + {%- if ethereum_node_el is defined +%} + attrs = push(attrs, {"key": "ethereum_el", "value": {"stringValue": "{{ ethereum_node_el }}"}}) + {%- endif +%} + . = { + "resource_log": { + "resource": {"attributes": attrs}, + "scopeLogs": [{ + "scope": {"name": "{{ ethereum_network_name }}-vector"}, + "logRecords": [{ + "timeUnixNano": ts_ns, + "severityNumber": sevnum, + "severityText": sevtext, + "body": {"stringValue": msg}, + "attributes": [{"key": "stream", "value": {"stringValue": strm}}] + }] + }] + } + } + ''' + + # Batch many shaped log events into one OTLP envelope (reduce works on logs). + [transforms.batch_envelope] + type = "reduce" + inputs = ["otel_shape"] + expire_after_ms = 30000 + end_every_period_ms = 5000 + max_events = 500 + merge_strategies.resource_log = "array" + + [transforms.finalize_envelope] + type = "remap" + inputs = ["batch_envelope"] + source = ''' + . = {"resourceLogs": .resource_log} + ''' + + [sinks.otlp_logs] + type = "opentelemetry" + inputs = ["finalize_envelope"] + [sinks.otlp_logs.protocol] + type = "http" + uri = "{{ otlp_endpoint }}/v1/logs" + method = "post" + encoding.codec = "otlp" + auth.strategy = "basic" + auth.user = "{{ ethereum_network_name }}" + auth.password = "{{ secret_loki.password }}" + # One event here is already a full OTLP envelope (built by the reduce above). + # max_events MUST be 1 — OTLP/HTTP allows one envelope per request. + batch.max_events = 1 + batch.timeout_secs = 5 + +# Static EL peers (erigon, ethrex, nimbus-el): mainnet peers have none of our blocks. +# Firewall EL p2p on every host right after -t init-server (before any EL starts): +# FW_HOSTS='ethereum_node:bootnode' scripts/shadowfork-ethrex-firewall.sh msf-2 +# The 27 geth/besu/reth/nethermind ELs, from the first playbook run's ethereum_node_fact_discovery_el_enode facts (2026-10-07). +shadowfork_el_static_enodes: + - enode://182aa43c0045e4dc6a59c55e08bf15b08d72646b8c96cf6de63b63e2ece6074eea440eb65340552a71db93c501028f4996dea8695ba02d52877236fde34e2220@165.22.214.79:30303 # bootnode-1 + - enode://4d4cd5e3a249e0e68277da4512307cd948fd56465546f466e92dcf03d9d1dc7dec837cc4882e36a61a63eedf6db0f0f9ab9bc1e784121a7a0643b04b0024be11@134.122.118.11:30303 # buildoor-lighthouse-geth-1 + - enode://0b66e95f8c543a4b4e4e16351ffdf9370b91ac36908e20c6cfe5d2457dbdc5314efada923bdb0f276885e1efb06ebbcbcf836fa9198b3ba07d1809086293a453@165.232.143.60:30303 # buildoor-teku-nethermind-1 + - enode://543215cc00c4705e26e32c684cd441984755611ff8d782d20595d2498d9d5d3bc51486f513fdf811f8bf47a294a3b72f21f4263f044a78366ff158598386b25e@162.243.160.238:30303 # grandine-besu-1 + - enode://861731a9793525326d38fae0fd2406b61383f025a7f035ff309de431785c3d0b7bdbc0e325c5e807ce0c6cad3dd5bda94608dce2a0f5f1dc5796a2267710f3ab@139.59.142.22:30303 # grandine-geth-1 + - enode://5d4dd99be4f0450a6dca4d5d5600a6ab8d3b6df3ff099b183a94b7117e3de25c4c04293dad52f9d088b94c5c85511ff56584069f00baed3b1e09a9b00821d159@137.184.152.222:30303 # grandine-nethermind-1 + - enode://1555805631467b69f511a1e6ab34701c3a367644878cbb8375fa22ae5640405f64c22885a5c2a629ce9b3ea87f92369d365a355a6969b769492799a8e2087ccb@165.232.180.28:30303 # grandine-reth-1 + - enode://2929d39acab48742bbffe21691f605aea7fcd6f47e7a2eb381857f35a6ddcc3f81ba9c17e150eda54d34eab8f1aad9a04d0c53f12b426bb8b1f7ebd3feb02dcc@146.190.143.108:30303 # lighthouse-besu-1 + - enode://1f7c633d559b09361b418885270946def404976d18cf31bfe6ec2a80d01c544c95697384e6da2879d98c4076ef882c3fa7ef365e88ee288c9e2a6637e8eb4ea3@134.209.177.39:30303 # lighthouse-geth-1 + - enode://d10489d056d158ad3387e025f6b61f220991dd79fe16a54c5362267b2e4975da78e723f4b9b6d44c3d74c5bcd2287c34318a95fb6baf02aa6226f07793eaf621@138.197.134.58:30303 # lighthouse-nethermind-1 + - enode://b4f270b1c07ee9568d0762741dd3b058fa22983da1a94195c99cc72ce5c1c0f5aaead0739fe08146801696d3913b69e85c91914b8149d297ed012c575f3335bb@165.22.42.44:30303 # lighthouse-reth-1 + - enode://62ce527ef2ec32df22ac6d884f0990bce7df26d602df610156dbe04fc0cb7a29b0063630e29185a65379924a42f9308490dbc69c25b9cd78e262a50a981b223f@201.79.6.41:30303 # lodestar-besu-1 + - enode://dd70c7191fdeae64c38f4b2bf6797575d2a9db9f244917af3e8f292a72ab323cdb06be7a46ebeffb7fb01d13fb8d31be7bce4214b8d8719ee5329b1bd617c3d8@188.166.82.109:30303 # lodestar-geth-1 + - enode://989de4d1d045e6e948246207ae98592a05f4c6e03f2b2a6a66f0ab891488205cdaafe6635e1e78cf3702b2252a9f8c60c04f049f8b11e73b3efa7eaa41e2cd1c@143.198.193.222:30303 # lodestar-nethermind-1 + - enode://767876e92d10f00f8a056ed3f77f8c46630005bd886066afd23659b710dde1d6d5f133469a8325a7849cfad860caa60a23e38f510ec1d291ac3493f94dfaf69c@159.65.186.218:30303 # lodestar-reth-1 + - enode://630ca24e819f30fb7f2fc748a942b03b0e242b4c9f485c685cceccbbdc84720d0589064270874de8e328a91f9a05b3b83264d1bbce0d04eb9275230cd6169ce9@159.223.78.111:30303 # nimbus-besu-1 + - enode://f42a3a4d5428c62dcc77bae868a630e0e167a6c93f13bb9ec77f7740edbcd0636896a6cdb112401fd6d95776d0eed9ac68a5a13c33fcbeaf0d911f2bd555fbbb@168.144.149.58:30303 # nimbus-geth-1 + - enode://c6338a35aa889787464156f80b4f966afeb383c12b8b0e96fe69d193d3fbfbdd31c73aba109624b9d26202c0401391e21fe0c37dc207bfa517450f6644165ff1@147.182.149.145:30303 # nimbus-nethermind-1 + - enode://c2bc8f2f5a84a43de9a4fd118f2fbf85e800ffbd2f09ba70d2cb1e3c655a961fb9e5c8c35a30bcfc6e098d7e0b7ba42c74cafd340b1284d9c78bd330929f33c5@159.203.86.74:30303 # nimbus-reth-1 + - enode://1a28067a6124ebd021bd36e112e5f7c3d29813972be7c8b40a9ae1a957c83c56e69502130620e701a68fff8d7b370ca012da7ef058f3738324d1a5b094bc1555@178.62.223.33:30303 # prysm-besu-1 + - enode://3b8a724028715338fd5813bbc0ca252391f93ae7899f9f5544234d667662e77cf662a85feda767bfd72e4e10764022767c67971417200cb8f12a646e746dc5db@139.59.12.219:30303 # prysm-geth-1 + - enode://c85042645cdf1e4e068d6ff0bce83555302c3e6b0d084d2bfafc82dbda43e64f29201a08642870cfcc91a03bd1cae70c07aa7c64e143a39d92433a3ad465a2e9@159.65.115.85:30303 # prysm-nethermind-1 + - enode://1daa8dd39ef039d4522c0ca3b83115bdad6062610485358b3d731610d6bd3b0d6fc82eb758445caa7ec6d2b16eb38a4ad7270f7fbc05dcdb88f9cc97be7709e2@143.110.218.78:30303 # prysm-reth-1 + - enode://ba7d05b484905ae4ccebcc96adde5111fcf5d88f2490f453190b2ea123dbb08370beb5639a30ee3dc12382134545efa2db3738d67dc884e4bd84d29a3a426594@138.197.121.104:30303 # teku-besu-1 + - enode://1095ea3a6f63612d1f32d403de348bb51c5867780485256b49cca69d6623495db8a87f0795b2fb69e217a2153077e468db1d2c1d3ca75e1fcfb9568b67a238e2@24.199.107.252:30303 # teku-geth-1 + - enode://9ac088d35ebd5908c665d9ff567e411747bad6124bfaa7a081e80f345be23e3cf94acfac44e110b2723ae728986b312bdf0b2083c1d46280c04c6d9cfc50d8aa@46.101.101.144:30303 # teku-nethermind-1 + - enode://6b9487acea9aa05c38adaa0fa61a0866564fc96797a754cf8b3d27fbbcad9d516484729812789b14d538303fe26377ec7c5c4d57c2bdba988d8c58caf665d6d7@178.128.203.221:30303 # teku-reth-1 diff --git a/ansible/inventories/msf-2/group_vars/all/images.yaml b/ansible/inventories/msf-2/group_vars/all/images.yaml new file mode 100644 index 00000000..96b9716a --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/all/images.yaml @@ -0,0 +1,68 @@ +--- +default_ethereum_client_images: + ### Consensus layer clients + lighthouse: registry-1.docker.io/sigp/lighthouse:v8.3.0-rc.0 + lodestar: registry-1.docker.io/chainsafe/lodestar:v1.49.0 + nimbus: registry-1.docker.io/statusim/nimbus-eth2:multiarch-v26.10.0 + prysm: registry-1.docker.io/offchainlabs/prysm-beacon-chain:v7.2.1 + prysm_validator: registry-1.docker.io/offchainlabs/prysm-validator:v7.2.1 + teku: registry-1.docker.io/consensys/teku:26.9.1 + grandine: registry-1.docker.io/sifrai/grandine:3.0.0-rc.0 + ### Execution layer clients + besu: registry-1.docker.io/hyperledger/besu:26.9.0 + geth: registry-1.docker.io/ethereum/client-go:v1.17.7 + erigon: registry-1.docker.io/erigontech/erigon:v3.7.1 + nethermind: registry-1.docker.io/nethermind/nethermind:2.1.0 + reth: ghcr.io/paradigmxyz/reth:v2.7.0 + nimbusel: registry-1.docker.io/statusim/nimbus-eth1:v0.4.2 + ethrex: ghcr.io/lambdaclass/ethrex:29.0.1 + +default_tooling_images: + mev_boost: registry-1.docker.io/ethpandaops/mev-boost:develop + mev_builder: registry-1.docker.io/ethpandaops/reth-rbuilder:develop + mev_relay: registry-1.docker.io/ethpandaops/mev-boost-relay:main + buildoor: registry-1.docker.io/ethpandaops/buildoor:main-latest + authenticatoor: registry-1.docker.io/ethpandaops/service-authenticatoor:main-latest + xatu_sentry: registry-1.docker.io/ethpandaops/xatu:1.24.1 + xatu_cannon: registry-1.docker.io/ethpandaops/xatu:latest + xatu_mimicry: registry-1.docker.io/ethpandaops/xatu:latest + xatu_cl_mimicry: registry-1.docker.io/ethpandaops/xatu:latest + xatu_relay_monitor: registry-1.docker.io/ethpandaops/xatu:latest + ethereum_metrics_exporter: registry-1.docker.io/ethpandaops/ethereum-metrics-exporter:latest + tx_fuzz: registry-1.docker.io/ethpandaops/tx-fuzz:latest + forkmon: registry-1.docker.io/skylenet/nodemonitor:darkmode + forky: registry-1.docker.io/ethpandaops/forky:latest + fauceth: registry-1.docker.io/skylenet/fauceth:fix_fee_estimation + powfaucet: registry-1.docker.io/pk910/powfaucet:v2.5.1 # fixed client bundle + 250000 ethTxGasLimit default + homepage: registry-1.docker.io/ethpandaops/ethereum-testnet-homepage:latest + checkpointz: registry-1.docker.io/ethpandaops/checkpointz:gloas-latest + blockscout: registry-1.docker.io/blockscout/blockscout:latest + blockscout_frontend: ghcr.io/blockscout/frontend:latest + beacon_metrics_gazer: registry-1.docker.io/dapplion/beacon-metrics-gazer:latest + eth_fauceth: registry-1.docker.io/chainflag/eth-faucet:latest + blobscan: registry-1.docker.io/blossomlabs/blobscan:latest + blobscan_indexer: registry-1.docker.io/blossomlabs/blobscan-indexer:latest + dora: registry-1.docker.io/ethpandaops/dora:master-latest + dugtrio: registry-1.docker.io/ethpandaops/dugtrio:bbusa-endpoint-failover + ethereum_genesis_generator: registry-1.docker.io/ethpandaops/ethereum-genesis-generator:6.2.0 + tracoor: registry-1.docker.io/ethpandaops/tracoor:0.0.2-glamsterdam-devnet-11 # pinned: floating tag + IfNotPresent won't re-pull + ncli: registry-1.docker.io/status-im/nimbus-eth2:unstable + lcli: registry-1.docker.io/ethpandaops/lighthouse:unstable + zcli: gloas + assertoor: registry-1.docker.io/ethpandaops/assertoor:master-latest + erpc: ghcr.io/erpc/erpc:latest + prometheus: registry-1.docker.io/prom/prometheus:v2.40.7 + node_exporter: registry-1.docker.io/prom/node-exporter:v1.5.0 + cl_bootnode: registry-1.docker.io/protolambda/eth2-bootnode:cleanup + json_rpc_snooper: registry-1.docker.io/ethpandaops/rpc-snooper:latest + nginx_proxy: registry-1.docker.io/library/nginx:alpine + nginx_proxy_gen: registry-1.docker.io/nginxproxy/docker-gen + nginx_proxy_acme: registry-1.docker.io/nginxproxy/acme-companion + nginx_proxy_cert_loader: registry-1.docker.io/ethpandaops/debian-docker:latest + nginx_proxy_cert_linker: registry-1.docker.io/nginxproxy/docker-gen + vector: registry-1.docker.io/timberio/vector:0.56.0-alpine + spamoor: registry-1.docker.io/ethpandaops/spamoor:master-latest + blobber: registry-1.docker.io/ethpandaops/blobber:latest + syncoor_web: ghcr.io/ethpandaops/syncoor-web:master + syncoor_server: ghcr.io/ethpandaops/syncoor:master + slashoor: registry-1.docker.io/ethpandaops/slashoor:master diff --git a/ansible/inventories/msf-2/group_vars/besu.yaml b/ansible/inventories/msf-2/group_vars/besu.yaml new file mode 100644 index 00000000..b62becd4 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/besu.yaml @@ -0,0 +1,102 @@ +--- +# role: ethpandaops.general.bootstrap +besu_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/besu?format=txt') | select | list) + + besu_extra_github_users }} + +# role: ethpandaops.general.snapshot_fetcher +snapshot_fetcher_out_dir: /data/besu + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: besu +# role: ethpandaops.general.besu +besu_container_name: execution +besu_container_image: "{{ default_ethereum_client_images.besu }}" +besu_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" + BESU_OPTS: "-Xmx16g -XX:MaxHeapFreeRatio=30 -XX:MinHeapFreeRatio=10" +besu_container_volumes: + - "{{ besu_datadir }}:/data" + - "{{ besu_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +besu_container_command_extra_args: + - --genesis-file=/network-config/genesis.json + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --discovery-mode=V5 + - --rpc-http-api=ADMIN,ETH,MINER,NET,TXPOOL,WEB3 + # Restored jochemnet snapshot: nothing to sync, FULL just follows the CL from the + # snapshot head (fusaka shadowforks). The snapshot may come from another besu + # version, so skip the datadir version check. + - --sync-mode=FULL + - --version-compatibility-protection=false + # nimbus-el has no snapshot and snap-syncs from the restored nodes; besu serves snap + # only when asked (geth and nethermind do by default). + - --snapsync-server-enabled=true + - --data-storage-format=BONSAI + - --bonsai-limit-trie-logs-enabled=false + - --color-enabled=false + - --cache-last-blocks=2048 + - --rpc-http-max-active-connections=1000 + # devnet8 block gas limit is 200M; raise the RPC cap so eth_call/estimateGas can simulate full-block txs + - --rpc-gas-cap=200000000 +besu_container_pull: true + +prometheus_config: | + global: + scrape_interval: 12s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/metrics" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/blobber.yaml b/ansible/inventories/msf-2/group_vars/blobber.yaml new file mode 100644 index 00000000..60f831ab --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/blobber.yaml @@ -0,0 +1,56 @@ +--- +blobber_container_image: "{{ default_tooling_images.blobber }}" + +blobber_log_level: trace + +ethereum_node_blobber_enabled: true + +# Storage configs +blobber_datadir: /data/blobber +blobber_validator_local_key_folder: "{{ ansible_inventory_sources[0] | dirname }}/files/validator_keys/{{ inventory_hostname }}" +blobber_validator_remote_key_folder: "{{ blobber_datadir }}/validators" + +blobber_container_volumes: + - "{{ blobber_datadir }}:/data" + +# Blobber specific configs +blobber_beacon_api: beacon:5052 +# blobber_slot_actions: {"name": "equivocating_blob_sidecars", "broadcast_blobs_first": true} +# blobber_slot_actions: {"name": "conflicting_blobs"} +blobber_slot_actions: { name: blob_gossip_delay, delay_milliseconds: 1500 } +blobber_slot_action_frequency: 1 + +blobber_container_ports: + - "{{ blobber_p2p_port }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port }}:{{ blobber_p2p_port }}/udp" + - "{{ blobber_p2p_port + 1 }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port + 1 }}:{{ blobber_p2p_port }}/udp" + - "{{ blobber_p2p_port + 2 }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port + 2 }}:{{ blobber_p2p_port }}/udp" + - "{{ blobber_p2p_port + 3 }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port + 3 }}:{{ blobber_p2p_port }}/udp" + - "{{ blobber_p2p_port + 4 }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port + 4 }}:{{ blobber_p2p_port }}/udp" + - "{{ blobber_p2p_port + 5 }}:{{ blobber_p2p_port }}" + - "{{ blobber_p2p_port + 5 }}:{{ blobber_p2p_port }}/udp" + - 127.0.0.1:{{ blobber_api_listen_port }}:{{ blobber_api_listen_port }} + +# - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-geth-1.{{ network_server_subdomain }}" +# - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-besu-1.{{ network_server_subdomain }}" +# - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-erigon-1.{{ network_server_subdomain }}" +# - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-nethermind-1.{{ network_server_subdomain }}" +# - "--cl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_beacon_prefix }}lighthouse-reth-1.{{ network_server_subdomain }}" +blobber_container_command: + - --cl={{ blobber_beacon_api }} + - --beacon-port-start={{ blobber_p2p_port }} + - --log-level={{ blobber_log_level }} + - --external-ip=0.0.0.0 + - --slot-action={{ blobber_slot_actions | to_json }} + - --slot-action-frequency={{ blobber_slot_action_frequency }} + - --validator-key-folder=/data/validators + - --validator-proxy-port-start={{ blobber_api_listen_port }} + - --enable-unsafe-mode + +blobber_container_command_extra_args: [] +# Default image pull policy +blobber_container_pull: false diff --git a/ansible/inventories/msf-2/group_vars/bootnode.sops.yaml b/ansible/inventories/msf-2/group_vars/bootnode.sops.yaml new file mode 100644 index 00000000..cd4ba32c --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/bootnode.sops.yaml @@ -0,0 +1,158 @@ +cl_bootnode_privkey: ENC[AES256_GCM,data:ywhGk8vFMF6ishOKGWMfSn0IxJttDiSBqaHgserxIamQvRMzJPcmMvAStVfMZB7XBHGyiWLy9vvDFhhQzBiocg==,iv:iEHtpLwDBPbJ3Ny/ZZLf+jX2nWunP21VhgU5D6fiSyQ=,tag:d2jUPi/+szwu4fkG1UuYKA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2025-01-31T05:18:36Z" + mac: ENC[AES256_GCM,data:OvHF2KM4DeBqRrBC2bhcpCrUbW2mwIvt+6+nwXld/qoDhgTxT8DBXGAfFY9wE172uElnV6O2bHGfgwCVX121Jg6d1rYzvcXrXuLaaTqLCTjy3xAkE0GrjJpXDAzhghsY95NU+n3u5Rz+49SffK/sR/dqRyNsASsW64eTzdMvXng=,iv:5di/nwLQieMrIxn6IEOP1UcJjick+X1l0kLka2e/72w=,tag:7jfPRLfWT2WiwllOfvxtEA==,type:str] + pgp: + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA32GcoRiZf6pARAAjH+IB4Omi4oRDCts8pOiPZpqGLhtexR36+2Wbgf850w5 + iHfN3I1jp/x+LhMkHF7OTjsUVCf27Q+NlAy5kv/Ptgf8Ks52N1rLzway5g+ssRWB + 7BwrRdNWvrBPVrQ3tXgHJlUyY4GfpkVFTHBZ0SP4g9VL1Ym7fK7n850XzbeNGJRx + LVXKrL8BqFa6Yunc4QP8megBAZTFLktUBPwZfSycs6aZKixRDiad/cb4JWXk2wl/ + byiFcYb1GynJbcLxjRdPqbkAk9EeDObXIatudLtNKB/iHnGst0KozBOwNHXD0v5N + 9kjJ++FlMHy205cMkUQj/ZVZgUgPhg2EOUvtUNOh160PIDqoDVQAcdKahjvgSF4h + sM0FPEigOu4qUncv9ckPjiyzT/vQPH3ZwDri1OMeOtpsBIx64kPZkx7w6Z60RaNQ + E9qa8tc/h4Co2NftEY9z6au5n9QN8J+b8z46DCbCKvfMRXffcUNZc9AcqeQ0add3 + I3xzdynbuQKbLLj2MRrBatrv+x1goEddXGXDfDaKmrYQJomsppYHuf2WOcamRLaW + Fx3FkMNnbcTzsGJQHHvRglOWacMeeSedQb55WePZUPO8P4im+vkihhqOX9J7Pufq + h7LNhub+IACZ02Tqrc8x53SsWSw9BDE67PpAPfhIopUrosUdy/dKRxYVWHFlpLjS + UQGmHup2nSh0C32VGJa5RdpFW6vTsgigKLS8nxULQMJ/9hxLrfpIRXLlVMxK89wk + bE5b1RsyNJ6UNXy5vI4gDmXpyXd7H5/QRDxPzLrf4WiipA== + =tx9b + -----END PGP MESSAGE----- + fp: 80DB2B4EF6CA4D9829C280605636BC0E08138A24 + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA87Wber4r4dGAQ//T+tC6M6C2NaDxpjRdsB2/+6bj9YhUIFL4EhDgasNYybW + ewY3/zTwfcVdWaz7hNup6NaYqkGlVTGLGEeGLD8LsKN1QXSQD+jrkVO3kbJ8juZf + hABQbOplI/x5IFz63qvid1A16xuxURCclo6GjL/Z8tpLjU/QYx5Y+jI7sXXeymxC + m9sg79pYBMyKyF4WoSJ7ftrIkwE93rEGJPjTQ76kismwV/RNf3dWIiypSLC6xdXD + d9DfCzN26iYK1GCONqYRDgOnHkvpFWvxXaUw6vmNm7uUUzRn/9x1mUJixL6GoY+9 + vTp6Qw4v534QjRAwG1G4lpv2GzliyKV6QI7AvhVoup3BzQBHnIUf7P3so0Ug33qK + cw2YvaNZosEuFFjaHUl+8Q+FIYXSc/ctG61eCeraqNnzToBzsb2P6uk2UDspy0Dk + lKTsqCyvW5GIPhVgAa8zBY24IEB85QfqWOdifZioRDbvl3x+IBozcTHwv8BxCoxm + p1VejU7lTlaEKt8QiQgi/ox1SRAQaMtICaG4uGBdLqb9fQxfolBZbNPfOlSSsTCn + DepMfCG31l2tuQQVUTo821fqsmQR9/zG3pzM3qgz+C8BPzovR6zronVASPoSNLvP + Pgk1S9N5/skbQ4jSqBiVyoyddkmEgHXlRsENueaj4Mobzoi9PX0PLjDAx911k+HS + UQHcYCMo6RecBliX4xFJ+Eu7h75QVgTe0o0ZeJnIzng9qfSX9qbpz5q7l1JtsWPV + 5P3tIxwWRrOoUhM35DGJNYsshFXx/jVZLQRB9hxxkJGekg== + =P8j2 + -----END PGP MESSAGE----- + fp: 69F66EEA7AE36CCB77DDB8CA1BC39532FB4A2DBD + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA4hw3nPn82LyAQ/+KXu4QcH7jEK0FOIkpPgFAKAxgTr7iodgHnyMF2orvyT4 + Kns5IKt9REOgbhZ6fKx07lcf3HUPe3rMPJ13Xy2dG8lRc3ucul/L7q2OR7+9vOVt + EC+AIokjSzEkyDtIvWXFArP20qmEyThcMWJWY531VTdrXtv2RhyH3hBwBFWpweMr + B3N5WCVcr90retI3k0dCgbiz7CK4tlUortc8fTQJtwo3FGbO3spojFMqf4DkU4ag + PGEhsWHPY6RyoxBQCFR2yWGWuV7ZSpF3KoKQBDYlXzSpBG82N96OppbqDl59P3KN + LGKSRIsvq4UNR+dge9jtOdkealEVfo4VSjDAfOx8ceUAUn6MAGSAvyK3xqd5AFc4 + rBTMavg2rNqfrJsHuSlnVstqi+WOd+c6JnEZHwyqHPGxUHzjUOjpO/u1cmkJk4+J + KhbeZwNlNsHpA0V3olYgZHOmyCya6y9IOhB5HJrNzKYbUwzIaEXygc1/AMvzNDye + d/xUI0p3BHglra8tC3FUStsO5WJnplc41YlcteQrWaI/lsh4FQFMhpBBRTYyXMDC + JZGGcS+VejZGh55kmMfVjc9jy1S6WcOQrYFk58ZYcW4M2/6xHVnI9u0RVI8pDNLb + sdiDF+NMMmpZfx7Bu51K4tzoW2zZeoYgkY7sf3feDEsQ1Fa8m1GzAqpmnnOaknDS + UQEZG9MTHeZNdAYIHpcX72L702w0VLx+w9WJSbt/HBYpdDPdfiHe4r1SWhlWBDHo + obhPTmQyl9BP77gRvv7YaVnbd1OzHyugVY9mXcPexKV76A== + =bvyI + -----END PGP MESSAGE----- + fp: D1002590180DE371BCB455EAEFCD4ACD0B4D3F6E + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAw01qfIBwY0JAQ/+JlPvzxbn8hp8QXmtxy18SsKYXxBQ4VAXpOsOgsKkqPXD + sGjeJtVGPal0u3avizQzuua+4wOD+3FLHM4Pyo0sgpIm5LG50shyerIC2qVnChBB + boxsy/qeYk7XYq7zgXAhoo9PE5rQX2jC+AZ28QOfP7K6g1yewJqLVuyS3uUt2HoI + r+EBHrict4RPd6klAhPNN+z813+kGP/ySkoqfkObFz25qorXk9YGqoQW+tfX5XCg + Ows2iX+/0m/u5SX30fTVnzbBnxabDerTQj5XPycdhWVR+YBeDIsTd1PxCk+xKrqr + CP4lXEcXkbuiOD8bCZ0Nfias9F/G4sYMYD4y+LMKe3yNcQ5VP5+JYCiBucl8OdTc + r75PxyciQZy6a8sctXQtG3HOuPNvDVi+OSIyFQ09HHYCC4y0j6yfW7YYP5Tv6vMm + yXy0lT/tc4IIEfFxHBQyNWNN6xZlHW0TBe+s0RQ2cnbAdKk9imfnhRF6+3KF0nsJ + WYXk5yShxxW3S1ujm4C+LZFv6bAZz1RN7krNFbeaMNZJkx6Yf9sVzZIOdcxik0oN + LDvnTuxXRLEBBGEgOBIbvVQHQGDZzdHmbaXlpKnnV3iEL9sjXl+qBVXIwzp+D5Nc + UM3WbuDs6LIr+O6XWqMZrhz+HSH2dKi8If72+DkCB+8HNZWNYLiWe8sqA/kEzt3S + UQGPRo12reWMKimS7gmCZSFmCkhAxzvbYdJP8ToTmxFKbCTxsItLnHoCbSD4S94W + tnwa5hhxezaxP1uvGFUfwvHPXMma9ChbFnZL1l2wuFaVUg== + =nxty + -----END PGP MESSAGE----- + fp: B9F81F327CF5346860E85269D7AF98F214C59E4E + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMAwDFOgk9tOQ+AQf/S3r7EsW1J1wcBKkrWws4xazvLUrkOPV7Pev4quKpSHG2 + d9pr45IFFW3zgqjqBpUzCzo+a9wXM2/IzHYe44wQSHmmhzXijmFARZ0TXMgtiAg+ + EJUdwrbhglemA0s1WPlRyGIBdVvkfUWFpb9EGMqsB9gqGo/JNmTyUiG3+47e8iUf + JTp2RbmdCSvDY78QiDF7r31Tw4Rrn+ep4FTMyUw/XY7539mz4D6/6pX01Tb7tGxV + V8yzYchD9kg3R3Jwgq2UcU9f9yGDa9epmo0qDfwb+ECpv1SsS41VSkGk9/KDTjr0 + 8LuhrsUg8S4ZCHVfEuOrs+s1RBq2tJU+esKXB9/sJNJRAWqYRPTSIy4Ytq7zT0DO + dATxseHf3gAhP9eBlKyBDvFcNPDp+Lhs7akJndrRm/8n/s3GvEJJAej3ur9ETOTI + 4BYMwVrwQkwtwGmrZXT+iXh1 + =sklM + -----END PGP MESSAGE----- + fp: 0600D41E1313E31016F7C55BF597BC0C5F22D1A2 + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAz4a8AV36xppARAAhikgDq91xe/hHoTK+nJjxjPe7HnabWcvaYSElOTcJdT4 + R2z8fgJJ8SRn+XSFIZZbufPChxhAga5L+KlO3d+9ifvHttDZ/wKPitMfUne1M9mX + SR+NcK1tKJn5iQJ9Bxx50l7u/ZFTnvDHfis37NDQnrU3osEQmefc+zc/YHfDtXYb + LU3HVqlSzciNpKLrJtYi1Y6l8BFgnnzxMyfUAn7gL16K6xh4wUxUOvesiQPltXRQ + bkEGvAakdOg6x9LBN8w9o2d0mV1/WADPKjPgyF2Rrgv7gU0ZpU2qracvUaIvCqfG + vg5Tj34NZAqc1Gaa7Y1tuRyijLJPT2FR0EH3Qd9SBvEeyAMjgQ1wbPuvwVioNpqg + sjXKALKL2O1RIPT2FTcbsPHUMlUyWU5J4HZ5sbTeZGDcYt9Izk7C1wJMvH1922iC + FUobJ52hAkUr05ensTqp36nbvU/7MIFiOB7uO4ro12UV90SyIPDZgjXBZYzAER8B + ggX56/zzjQyUzfA8z2NWzSj21WY45d3e+rXKU7MIlmvlqG21jyIN7nizzCQkQxIm + 8kwEAuPJHS2nDbewA2dgWF4uTCPq9BH1QWYD4az4bxtuuF56O2C/zIN6m90UmLoU + U840pAg4jq7EWiDSr9Y8Bop2E3rXnY8bKHvoj8ouLH2CE3M6ozC08qxORKCg20LS + UQGLDqG3LoDHmupHH5OBw7GdWg4mhnfn1fXW/3mvXeC6+lY43BGxHBeJV6MIXFDn + winRBpyrrOpf7JyDa6evUuhrYzckgiatZjLYPKUqXdyVuQ== + =QKuz + -----END PGP MESSAGE----- + fp: 29C50D01122FDE78E257482DAA497EB2610A8435 + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAxYzhHRfYJYtAQ/5AXwnZoyGM43yGkK1zVhO9JBz3B/ZhOHtReRbQCHoET6P + Mi9KuzuDWbXLq90z0dKMY1vDTJCHOThsQiqiR4LVJCiFHuDdgCpkVBGA+aoA8QgZ + DdynxVKXbaIQRC3Cc1rDe/lFDfsMeibebsCgxX3rWytho75DvkPVgKqakCk6qZJp + Z3wD0MH7DWps1A+fB+Q+W+n6uZFxIJ8OtPVdSU4H7FZ42WkxZZ3xxygJsVYzUdT2 + 7qWqArs3K7HRTO2Tx0kssGUo0IYTvrYwWSJjDyqQCe5hATzqY7ailzttiWNtM7Tl + XCkwb+QpvAkTFVwFdI+meZEVgme//+ZM+5slTg2ACALG+diwzDwca5EQtKheMdN5 + BBrQuh08Mnz5QJA3mLtvRFpYeogaTb82EaoXvm+BlydUKrdL+kuuK9AoeF2CvYuy + or1zc2nrfeTCRWRtWlfeJxdBkNMkxnVFwnhik5jvQmCnKZdZboWZIiTO5ABeRpCz + peh7dwLIK92GPLHWYsvKjGsINuDxawpX4iOCJSgg1JlzDp/Qim5hjCaTcrfUXXOl + WNK5gUt56Ij8Uk0GEmRhECgmKEuByLEpOKYeG0nLBopkNxKWd8RUvK8GCx+nlb+s + 7KmlrzvpqqLpXFJ5u0PjiRiw0dOJhSahwK+yROE8qQh3nVsJlnWDnjvcfh1g2BnS + UQH8UO8RDIVvocBWXnNYXAoSbv2UVE10HxphyAmMmgXB4Su2M4yXajMg6TlWtkZq + tHX+uD/SdA44VImJmsdE4AwP+yODWSYgj98ziSLWAKTPBA== + =Gc8d + -----END PGP MESSAGE----- + fp: 9BE537027CB7467923E240FF2AED09371C121F91 + - created_at: "2025-09-16T09:19:21Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DjS7VXZx9i8YSAQdA+QaCeWae6exqCLoTbViPC5MHCVuc5FHuJBfgKpTDzBcw + z666dWfo9PmP7os82CM0gvMSEwoKihm0z8Khux3yGGQFXgQlTavPCsmnEOBtqntH + 1FsBCQIQq8ASFMaLcLuUJQi4Jn7B4LJJt692Venh5ajhah5uOet0IHLxaTsYxpTL + VDFruyur7UAz73in3NcDKQsdsVly/1YvAqUDUNQZCR46JD1a6inxBvNSy8nn + =kExY + -----END PGP MESSAGE----- + fp: F93098C3D8ED15D0924A6DDAEB4E93A88660C55B + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/ansible/inventories/msf-2/group_vars/bootnode.yaml b/ansible/inventories/msf-2/group_vars/bootnode.yaml new file mode 100644 index 00000000..5680a8de --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/bootnode.yaml @@ -0,0 +1,181 @@ +--- +# ethereum_cl_bootnode: "{{ hostvars[primary_bootnode]['bootnodoor_fact_enr'] }}" +# ethereum_el_bootnode: "{{ hostvars[primary_bootnode]['bootnodoor_fact_el_enr'] }}" + +ethereum_cl_bootnodes: >- + {{ + ( + ['lighthouse-geth-1', 'teku-nethermind-1', 'nimbus-besu-1', 'prysm-reth-1'] + | map('extract', hostvars, ['ethereum_node_fact_discovery_cl_enr']) + | select('defined') + | list + ) + }} + +ethereum_el_bootnodes: >- + {{ + ( + ['lighthouse-geth-1', 'teku-nethermind-1', 'nimbus-besu-1', 'prysm-reth-1'] + | map('extract', hostvars, ['ethereum_node_fact_discovery_el_enr']) + | select('defined') + | list + ) + }} + + + +# role: eth_inventory_web +eth_inventory_web_container_networks: "{{ docker_networks_shared }}" +eth_inventory_web_container_env: + VIRTUAL_HOST: "{{ server_fqdn }}" + VIRTUAL_PORT: "80" + VIRTUAL_PATH: /meta/api + VIRTUAL_DEST: / + LETSENCRYPT_HOST: "{{ server_fqdn }}" + +# role: ethpandaops.general.bootnodoor +bootnodoor_privkey: >- + {{ + (secret_bootnodoor_seed ~ ':' ~ ethereum_genesis_chain_id|string) + | hash('sha256') + }} +bootnodoor_set_facts: true +bootnodoor_el_enabled: true +bootnodoor_cl_enabled: true + +bootnodoor_container_name: bootnodoor +bootnodoor_container_image: ethpandaops/bootnodoor:master +bootnodoor_container_networks: "{{ docker_networks_shared }}" +bootnodoor_p2p_port: 9010 +bootnodoor_ui_port: 8004 +bootnodoor_enr_ip: "{{ ansible_host }}" + +# genesis.json with timestamp 1, written by playbook.yaml: bootnodoor rejects mainnet's 0. +bootnodoor_el_config: /data/el-genesis.json +# Fork ids hash the real genesis; deposit_contract_block_hash.txt is the shadowfork head. +bootnodoor_el_genesis_hash: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" +bootnodoor_cl_config: /network-config/config.yaml +bootnodoor_cl_gvr: /network-config/genesis_validators_root.txt +bootnodoor_container_volumes_extra: + - "{{ eth_testnet_config_dir }}:/network-config:ro" + +bootnodoor_container_command_extra_args: [] +bootnodoor_container_env: + VIRTUAL_HOST: bootnodoor-{{ server_fqdn }} + VIRTUAL_PORT: "{{ bootnodoor_ui_port | string }}" + LETSENCRYPT_HOST: bootnodoor-{{ server_fqdn }} + +# role: ethpandaops.general.ethereum_node +ethereum_node_cl: prysm +ethereum_node_cl_validator_enabled: false + +prysm_container_tty: true +prysm_pprof_enabled: true +prysm_validator_container_tty: true +# role: ethpandaops.general.prysm +prysm_container_name: beacon +prysm_validator_container_name: validator +prysm_container_image: "{{ default_ethereum_client_images.prysm }}" +prysm_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +prysm_container_volumes: + - "{{ prysm_datadir }}:/data" + - "{{ prysm_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +prysm_container_entrypoint: + - /app/cmd/beacon-chain/beacon-chain +prysm_container_command_extra_args: >- + {{ prysm_container_command_extra_simple_args + prysm_container_command_extra_bootnode_args }} +prysm_container_command_extra_simple_args: + - --grpc-gateway-corsdomain=* + - --chain-config-file=/network-config/config.yaml + - --genesis-state=/network-config/genesis.ssz + - --contract-deployment-block={{ ethereum_network_deposit_contract_block }} + - --min-sync-peers=1 + - --verbosity=debug + - --subscribe-all-subnets +prysm_container_command_extra_bootnode_args: >- + {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }} +# role: ethpandaops.general.ethereum_node +ethereum_node_el: geth + +# role: ethpandaops.general.snapshot_fetcher (bootnode is not in the geth group) +snapshot_fetcher_out_dir: /data/geth/geth + +# role: ethpandaops.general.geth +geth_container_name: execution +geth_container_image: "{{ default_ethereum_client_images.geth }}" +geth_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" +geth_container_volumes: + - "{{ geth_datadir }}:/data" + - "{{ geth_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +geth_container_command_extra_args: + - --override.genesis=/network-config/genesis.json + - --http.api=eth,net,web3,admin,txpool + - --http.vhosts=* + - --networkid={{ ethereum_network_id }} + - --syncmode=full + # Without --bootnodes geth falls back to mainnet's bootnodes (empty is fine: none). + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + # no --gcmode=archive: the restored jochemnet snapshot is a pruned datadir +geth_container_pull: true +# role: ethpandaops.general.prometheus +prometheus_remote_push_url: https://victoriametrics-public.analytics.production.platform.ethpandaops.io/insert/1/prometheus # victoriametrics.ethdevops.io no longer resolves +prometheus_remote_write_username: "{{ ethereum_network_name }}" # gateway tags ingress_user=; the sops one is devnet-8's +prometheus_remote_write_password: "{{ secret_prometheus_remote_write.password }}" +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/buildoor.yaml b/ansible/inventories/msf-2/group_vars/buildoor.yaml new file mode 100644 index 00000000..055d06d4 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/buildoor.yaml @@ -0,0 +1,83 @@ +--- +# NOTE: precedence over sibling groups (prysm/ethrex) is established via +# `[buildoor:vars] ansible_group_priority=100` in inventory.ini — that variable +# is consulted while loading group_vars, so it cannot be set here. + +buildoor_container_image: "{{ default_tooling_images.buildoor }}" +buildoor_container_pull: true +buildoor_container_networks: "{{ docker_networks_shared }}" + +buildoor_jwt_path: /data/execution-auth.secret +buildoor_cl_client: http://beacon:{{ ethereum_node_cl_ports_http_beacon }} +buildoor_el_engine_api: http://execution:{{ ethereum_node_el_ports_engine }} +buildoor_el_rpc: http://execution:{{ ethereum_node_el_ports_http_rpc }} + +# Avoid conflict with prysm metrics on 127.0.0.1:8080 +buildoor_api_port: 8085 + +# Verify API tokens against the centralized authenticatoor service for this devnet. +buildoor_auth_provider_url: https://auth.{{ network_subdomain }} +buildoor_overview_url: https://buildoor.{{ network_subdomain }} + +# Wallet ECDSA private key +buildoor_wallet_privkey: "{{ secret_buildoor_wallet_privkey }}" +# Builder BLS private key +buildoor_builder_mnemonic: "{{ secret_buildoor_builder_mnemonic }}" +buildoor_builder_mnemonic_index: "{{ builder_index }}" + +buildoor_container_command_extra_args: + - --validator-ranges-url=https://config.{{ network_subdomain }}/api/v1/nodes/validator-ranges + - --lifecycle + # ePBS bidding/revealing is off unless enabled (devnet-8 toggled it at runtime via the + # authenticated web UI and kept it in the state DB); a fresh DB never bids. + - --epbs-enabled + - --slot-result-retention-epochs=1600 + - --slot-artifact-retention-epochs=1600 +buildoor_container_env: + VIRTUAL_HOST: api-{{ server_fqdn }} + VIRTUAL_PORT: "{{ buildoor_api_port | string }}" + LETSENCRYPT_HOST: api-{{ server_fqdn }} + BUILDOOR_INJECT_HEAD_HTML: + +# Re-apply the prysm.yaml merge so we don't lose the bootnode args while adding --prepare-all-payloads and --subscribe-all-subnets +prysm_container_command_extra_args: >- + {{ prysm_container_command_extra_simple_args + + prysm_container_command_extra_bootnode_args + + ['--prepare-all-payloads', '--subscribe-all-subnets'] }} +lighthouse_container_command_extra_args: >- + {{ lighthouse_container_command_extra_simple_args + + ['--always-prepare-payload', '--subscribe-all-subnets', '--import-all-attestations'] }} +lodestar_container_command_extra_args: >- + {{ lodestar_container_command_extra_simple_args + + ['--emitPayloadAttributes', '--subscribeAllSubnets'] }} +teku_container_command_extra_args: >- + {{ teku_container_command_extra_simple_args + + ['--Xfork-choice-updated-always-send-payload-attributes=true', '--p2p-subscribe-all-subnets-enabled'] }} +grandine_container_command_extra_args: >- + {{ grandine_container_command_extra_simple_args + + ['--features=AlwaysPrepareExecutionPayload', '--subscribe-all-subnets'] }} +nimbus_container_command_extra_args: >- + {{ nimbus_container_command_extra_simple_args + + nimbus_container_command_extra_bootnode_args + + ['--subscribe-all-subnets'] }} + + +ethereum_node_docker_watchtower_containers_list: + - execution + - beacon + - validator + - xatu-sentry + - buildoor + +# 16 GiB buildoor tier (vs 32 GiB matrix): override group_vars/ethrex.yaml so ethrex fits in RAM. +# - --syncmode=snap : avoids full-sync batch commit-ahead (the #7127 restart hazard) and large diff-layer backlog +# - --rocksdb.block-cache-size=4294967296 : cap the 12 GiB default RocksDB block cache to 4 GiB +ethrex_container_command_extra_args: + - --network=/network-config/genesis.json + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --p2p.discv4=false + - --p2p.discv5=true + - --http.api=eth,net,web3,admin + - --syncmode=snap + - --rocksdb.block-cache-size=4294967296 diff --git a/ansible/inventories/msf-2/group_vars/dns_server.yaml b/ansible/inventories/msf-2/group_vars/dns_server.yaml new file mode 100644 index 00000000..3155e2e3 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/dns_server.yaml @@ -0,0 +1,93 @@ +--- +# role: ethpandaops.general.dns_server +dns_server_disable_systemd_resolved: true +dns_server_is_master: "{{ inventory_hostname == primary_bootnode }}" +dns_server_acme_zone: "{{ network_server_subdomain }}" + +# One or more master IPs (for slaves to pull from / accept NOTIFY from) +dns_server_master: "{{ ([hostvars[primary_bootnode].ansible_host] if (groups.get('bootnode') is defined and primary_bootnode in groups['bootnode']) else []) | list + }}" + +# All slave IPs (for master's also-notify / allow-transfer) +dns_server_slave: "{{ (groups.get('bootnode', []) | difference([primary_bootnode])) | map('extract', hostvars, 'ansible_host') | list }}" + +dns_server_zones: + - zone: "{{ network_server_subdomain }}" + content: | + $TTL 120 + @ IN SOA {{ server_fqdn }}. zonemaster.{{ domain }}. ( + 00000000000000 ; Serial + 2H ; Refresh + 1H ; Retry + 1W ; Expire + 5m ) ; NX (TTL Negativ Cache) + IN NS {{ server_fqdn }}. + {# extra NS for each secondary, if their FQDNs are known #} + {% for h in groups['bootnode'] | sort if h != primary_bootnode %} + {% if hostvars[h].server_fqdn is defined %} + IN NS {{ hostvars[h].server_fqdn }}. + {% endif %} + {% endfor %} + + ; certificates + cert IN A {{ hostvars[primary_bootnode]['ansible_host'] }} + + ; bootnodes + {% for host in groups['bootnode'] | sort %} + {{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {{ ethereum_node_rpc_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {{ ethereum_node_beacon_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + bootnodoor-{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {% if hostvars[host]['ipv6'] is defined %} + {{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ hostvars[host]['ipv6'] }} + {% set proxy_ipv6 = hostvars[host].get('docker_nginx_proxy_public_ipv6', hostvars[host]['ipv6']) %} + {{ ethereum_node_rpc_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ proxy_ipv6 if proxy_ipv6 | length > 0 else hostvars[host]['ipv6'] }} + {{ ethereum_node_beacon_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ proxy_ipv6 if proxy_ipv6 | length > 0 else hostvars[host]['ipv6'] }} + {% endif %} + {% endfor %} + + ; ethereum_nodes + {% for host in groups['ethereum_node'] | sort %} + {{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {{ ethereum_node_rpc_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {{ ethereum_node_beacon_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {% if hostvars[host]['ipv6'] is defined %} + {{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ hostvars[host]['ipv6'] }} + {% set proxy_ipv6 = hostvars[host].get('docker_nginx_proxy_public_ipv6', hostvars[host]['ipv6']) %} + {{ ethereum_node_rpc_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ proxy_ipv6 if proxy_ipv6 | length > 0 else hostvars[host]['ipv6'] }} + {{ ethereum_node_beacon_prefix }}{{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ proxy_ipv6 if proxy_ipv6 | length > 0 else hostvars[host]['ipv6'] }} + {% endif %} + {% endfor %} + + ; mev-relay + {% for host in groups.get('mev_relay', []) | sort %} + {{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {% if hostvars[host]['ipv6'] is defined %} + {{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ hostvars[host]['ipv6'] }} + {% endif %} + {% endfor %} + + ; buildoor + {% for host in groups.get('buildoor', []) | sort %} + api-{{ hostvars[host]['inventory_hostname'] }} IN A {{ hostvars[host]['ansible_host'] }} + {% if hostvars[host]['ipv6'] is defined %} + {% set proxy_ipv6 = hostvars[host].get('docker_nginx_proxy_public_ipv6', hostvars[host]['ipv6']) %} + api-{{ hostvars[host]['inventory_hostname'] }} IN AAAA {{ proxy_ipv6 if proxy_ipv6 | length > 0 else hostvars[host]['ipv6'] }} + {% endif %} + {% endfor %} + +# role: ethpandaops.general.wildcard_cert_issuer +wildcard_cert_issuer_enabled: "{{ inventory_hostname == primary_bootnode }}" +wildcard_cert_issuer_base_domain: "{{ network_server_subdomain }}" +wildcard_cert_issuer_acme_directory: https://acme.zerossl.com/v2/DV90 +wildcard_cert_issuer_acme_eab_kid: "{{ secret_zerossl.ACME_EAB_KID }}" +wildcard_cert_issuer_acme_eab_hmac_key: "{{ secret_zerossl.ACME_EAB_HMAC_KEY }}" +wildcard_cert_issuer_email: ssl@ethpandaops.io +wildcard_cert_issuer_publish_port: 8080 +wildcard_cert_issuer_publish_psk: "{{ secret_cert_encryption_psk }}" +wildcard_cert_issuer_container_env: + VIRTUAL_HOST: cert.{{ network_server_subdomain }} + VIRTUAL_PORT: "8080" + HTTPS_METHOD: nohttps +wildcard_cert_issuer_container_networks: "{{ docker_networks_shared }}" +wildcard_cert_issuer_rfc2136_server: 172.17.0.1 # dns container runs on host network diff --git a/ansible/inventories/msf-2/group_vars/erigon.yaml b/ansible/inventories/msf-2/group_vars/erigon.yaml new file mode 100644 index 00000000..3577d7c0 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/erigon.yaml @@ -0,0 +1,113 @@ +--- +# role: ethpandaops.general.bootstrap +erigon_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/erigon?format=txt') | select | list) + + erigon_extra_github_users }} + +# role: ethpandaops.general.snapshot_fetcher +snapshot_fetcher_out_dir: /data/erigon +# Frozen up to 24401000; retire starts ~300 blocks after genesis. If it stalls then (v3.7.1 +# retire deadlock), add --snap.stop via erigon_host_extra_args. +snapshot_fetcher_filename: snapshot-pruned.tar.zst + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: erigon + +# role: ethpandaops.general.erigon +erigon_container_name: execution +erigon_container_image: "{{ default_ethereum_client_images.erigon }}" +erigon_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" + SAVE_HEAP_PROFILE: "true" +# Per-host additions go in erigon_host_extra_args (host_vars); --staticpeers only once the +# enodes are known (an empty one is invalid, and the old --nodiscover=false fallback made +# erigon v3.7.1 exit: "Flags --discovery.v4, --nodiscover can't be used at the same time"). +erigon_container_command_extra_args: >- + {{ erigon_shadowfork_args + + ((['--staticpeers=' ~ (shadowfork_el_static_enodes | join(','))]) if shadowfork_el_static_enodes | length > 0 else []) + + (erigon_host_extra_args | default([])) }} +erigon_shadowfork_args: + - --http.api=eth,erigon,engine,web3,net,txpool,admin + - --http.vhosts=* + - --ws + - --networkid={{ ethereum_network_id }} + - --db.size.limit=2TB + # Shadowfork hook: makes the erigon role re-run `init` over the restored chaindata + # and stops erigon from swapping our genesis.json config for its built-in mainnet one. + - --keep.stored.chain.config=true + # Otherwise caplin starts and follows the real chain (sepsf-1). + - --externalcl + # Chain id 1 injects the mainnet DNS tree whatever --discovery.dns says; only --nodiscover + # stops it (exclusive with --discovery.v4/v5 and unused --bootnodes). Peers: --staticpeers. + - --nodiscover + # Never fetch or seed mainnet snapshot files: jochemnet's segments carry mainnet names. + - --no-downloader + - --log.console.verbosity=debug + # devnet8 block gas limit is 200M; raise the RPC cap so eth_call/estimateGas can simulate full-block txs + - --rpc.gascap=200000000 + # pprof HTTP server (fetch profiles via port-forward / container IP) + - --pprof + - --pprof.addr=0.0.0.0 + - --pprof.port=6060 +erigon_init_custom_network: true +erigon_init_custom_network_genesis_file: "{{ eth_testnet_config_dir }}/genesis.json" + +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/debug/metrics/prometheus" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/ethereum_node.yaml b/ansible/inventories/msf-2/group_vars/ethereum_node.yaml new file mode 100644 index 00000000..1f2916c1 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/ethereum_node.yaml @@ -0,0 +1,118 @@ +--- +ethereum_cl_bootnodes: >- + {{ + ( + [primary_bootnode] + | map('extract', hostvars, ['bootnodoor_fact_enr']) + | select('defined') + | reject('equalto', '') + | list + ) + + + ( + [primary_bootnode, 'lighthouse-geth-1', 'prysm-geth-1', 'teku-nethermind-1', 'nimbus-besu-1', 'prysm-reth-1', 'teku-erigon-1'] + | map('extract', hostvars, ['ethereum_node_fact_discovery_cl_enr']) + | select('defined') + | reject('equalto', '') + | list + ) + }} + +ethereum_el_bootnodes: >- + {{ + ( + [primary_bootnode] + | map('extract', hostvars, ['bootnodoor_fact_el_enr']) + | select('defined') + | reject('equalto', '') + | list + ) + + + ( + [primary_bootnode, 'lighthouse-geth-1', 'prysm-geth-1', 'teku-nethermind-1', 'nimbus-besu-1', 'prysm-reth-1', 'teku-erigon-1'] + | map('extract', hostvars, ['ethereum_node_fact_discovery_el_enr']) + | select('defined') + | reject('equalto', '') + | list + ) + }} + + +ethereum_node_xatu_sentry_enabled: true + +# role: ethpandaops.general.brakebear +brakebear_download_speed: "{{ ethereum_node_cl_supernode_enabled | ternary('1000Mbps', '100Mbps') }}" +brakebear_upload_speed: "{{ ethereum_node_cl_supernode_enabled | ternary('1000Mbps', '50Mbps') }}" +brakebear_config: | + log_level: "info" + docker_containers: + - name: "execution" + download_rate: {{ brakebear_download_speed }} + upload_rate: {{ brakebear_upload_speed }} + exclusions: + private-networks: true + docker-networks: + names: ["*"] + - name: "beacon" + download_rate: {{ brakebear_download_speed }} + upload_rate: {{ brakebear_upload_speed }} + exclusions: + private-networks: true + docker-networks: + names: ["*"] + + +# role: ethpandaops.general.prometheus +prometheus_remote_push_url: https://victoriametrics-public.analytics.production.platform.ethpandaops.io/insert/1/prometheus +prometheus_remote_write_username: "{{ ethereum_network_name }}" # gateway tags ingress_user=; the sops one is devnet-8's +prometheus_remote_write_password: "{{ secret_prometheus_remote_write.password }}" +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/ethrex.yaml b/ansible/inventories/msf-2/group_vars/ethrex.yaml new file mode 100644 index 00000000..8b1c1caf --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/ethrex.yaml @@ -0,0 +1,98 @@ +--- +# role: ethpandaops.general.bootstrap +ethrex_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/ethrex?format=txt') | select | list) + + ethrex_extra_github_users }} + +ethereum_node_fact_discovery_el_key_cmd: + ethrex: xxd -p -c32 {{ ethrex_datadir | default('/data/ethrex') }}/chain-{{ ethereum_network_id | trim }}/node.key + +# role: ethpandaops.general.snapshot_fetcher (bare RocksDB; ethrex 29 reads /chain-, +# hardcoded: genesis.json does not exist at restore time). The May db is migrated on start. +snapshot_fetcher_out_dir: /data/ethrex/chain-1 + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: ethrex +# role: ethpandaops.general.ethrex +ethrex_container_name: execution +ethrex_container_image: "{{ default_ethereum_client_images.ethrex }}" +ethrex_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" + RUST_LOG: info +ethrex_container_volumes: + - "{{ ethrex_datadir }}:/data" + - "{{ ethrex_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +ethrex_container_command_extra_args: + - --network=/network-config/genesis.json + # Peer only with our ELs (mainnet peers have none of our blocks); p2p firewalled (SF-P2P). + # Until the static enodes are known, discover via our bootnode ENRs (discv5); once known, those + # enodes via discv4. ethrex (v29.0.0-rc.1) only seeds its peer table with --bootnodes when a + # discovery protocol is on, so turning both off leaves it with 0 peers. The p2p firewall on the + # ethrex hosts (scripts/shadowfork-ethrex-firewall.sh) limits port-30303 peers to our droplets. + - "--bootnodes={{ (shadowfork_el_static_enodes if shadowfork_el_static_enodes | length > 0 else ethereum_el_bootnodes) | join(',') }}" + - "--p2p.discv4={{ 'true' if shadowfork_el_static_enodes | length > 0 else 'false' }}" + - "--p2p.discv5={{ 'false' if shadowfork_el_static_enodes | length > 0 else 'true' }}" + - --http.api=eth,net,web3,admin + # Restored: follow the CL (default is snap). + - --syncmode=full + +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/metrics" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/geth.yaml b/ansible/inventories/msf-2/group_vars/geth.yaml new file mode 100644 index 00000000..fb2f09d0 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/geth.yaml @@ -0,0 +1,93 @@ +--- +# role: ethpandaops.general.bootstrap +geth_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/geth?format=txt') | select | list) + + geth_extra_github_users }} + +# role: ethpandaops.general.snapshot_fetcher (snapshot layout: chaindata/ at the top) +snapshot_fetcher_out_dir: /data/geth/geth + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: geth + +# role: ethpandaops.general.geth +geth_container_name: execution +geth_container_image: "{{ default_ethereum_client_images.geth }}" +geth_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" +geth_container_volumes: + - "{{ geth_datadir }}:/data" + - "{{ geth_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +geth_container_command_extra_args: + - --override.genesis=/network-config/genesis.json + - --http.api=eth,net,web3,admin,txpool + - --http.vhosts=* + - --networkid={{ ethereum_network_id }} + - --syncmode=full + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --discovery.v4=false + - --discovery.v5=true + # devnet8 block gas limit is 200M; raise the RPC cap so eth_estimateGas/eth_call can simulate full-block txs + - --rpc.gascap=200000000 +geth_container_pull: true + +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/debug/metrics/prometheus" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/grandine.yaml b/ansible/inventories/msf-2/group_vars/grandine.yaml new file mode 100644 index 00000000..4dfcb4cf --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/grandine.yaml @@ -0,0 +1,55 @@ +--- +# role: ethpandaops.general.bootstrap +grandine_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/grandine?format=txt') | select | list) + + grandine_extra_github_users }} + +# role: geerlingguy.docker +docker_daemon_options: + log-driver: json-file + log-opts: + max-size: 500m + max-file: "40" + +# role: validator_keys — grandine runs validators in-process in the beacon +# container (no separate VC), so reshuffles stop/start the beacon. +validator_keys_container_name: beacon +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/teku-keys/" + dest: "{{ grandine_validator_datadir }}/keys/" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/teku-secrets/" + dest: "{{ grandine_validator_datadir }}/secrets/" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ grandine_validator_datadir }}/change_operations.json" + owner: grandine + group: grandine +# role: ethpandaops.general.ethereum_node +ethereum_node_cl: grandine + +# role: ethpandaops.general.grandine +grandine_container_name: beacon +grandine_container_image: "{{ default_ethereum_client_images.grandine }}" +grandine_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" + GRANDINE_LOG: fork_choice_control=debug,p2p=debug +grandine_container_volumes: + - "{{ grandine_datadir }}:/data" + - "{{ grandine_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +grandine_container_command_extra_args: >- + {{ grandine_container_command_extra_simple_args }} +grandine_container_command_extra_simple_args: + - --configuration-directory=/network-config/ + - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }} + - --graffiti={{ ansible_hostname }} + - --features=LogHttpRequests + +grandine_validator_container_volumes: + - "{{ grandine_validator_datadir }}:/validator-data" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +grandine_validator_datadir: /data/grandine-validator diff --git a/ansible/inventories/msf-2/group_vars/lighthouse.yaml b/ansible/inventories/msf-2/group_vars/lighthouse.yaml new file mode 100644 index 00000000..430ff109 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/lighthouse.yaml @@ -0,0 +1,53 @@ +--- +# role: ethpandaops.general.bootstrap +lighthouse_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/lighthouse?format=txt') | select | list) + + lighthouse_extra_github_users }} + +# role: validator_keys +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/keys/" + dest: "{{ lighthouse_validator_datadir }}/keys/" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/secrets/" + dest: "{{ lighthouse_validator_datadir }}/secrets/" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ lighthouse_validator_datadir }}/change_operations.json" + owner: lighthouse + group: lighthouse + +# role: ethpandaops.general.ethereum_node +ethereum_node_cl: lighthouse + +# role: ethpandaops.general.lighthouse +lighthouse_container_name: beacon +lighthouse_validator_container_name: validator +lighthouse_container_image: "{{ default_ethereum_client_images.lighthouse }}" +lighthouse_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +lighthouse_container_volumes: + - "{{ lighthouse_datadir }}:/data" + - "{{ lighthouse_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +lighthouse_container_command_extra_args: >- + {{ lighthouse_container_command_extra_simple_args }} +lighthouse_container_command_extra_simple_args: + - --testnet-dir=/network-config + - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }} + - --suggested-fee-recipient={{ lighthouse_validator_fee_recipient }} + - --ignore-ws-check + - --enable-fast-confirmation + - --debug-level=debug +lighthouse_validator_container_volumes: + - "{{ lighthouse_validator_datadir }}:/validator-data" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +lighthouse_validator_container_command_extra_args: + - --testnet-dir=/network-config + - --graffiti={{ ansible_hostname }} + - --builder-proposals +lighthouse_validator_datadir: /data/lighthouse-validator +lighthouse_container_pull: true diff --git a/ansible/inventories/msf-2/group_vars/lodestar.yaml b/ansible/inventories/msf-2/group_vars/lodestar.yaml new file mode 100644 index 00000000..d041953f --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/lodestar.yaml @@ -0,0 +1,55 @@ +--- +# role: ethpandaops.general.bootstrap +lodestar_extra_github_users: # handles not yet in the coredevs registry + - lodekeeper +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/lodestar?format=txt') | select | list) + + lodestar_extra_github_users }} + +# role: validator_keys +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/keys/" + dest: "{{ lodestar_validator_datadir }}/keys/" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/secrets/" + dest: "{{ lodestar_validator_datadir }}/secrets/" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ lodestar_validator_datadir }}/change_operations.json" + owner: lodestar + group: lodestar + +# role: ethpandaops.general.ethereum_node +ethereum_node_cl: lodestar + +# role: ethpandaops.general.lodestar +lodestar_container_name: beacon +lodestar_validator_container_name: validator +lodestar_container_image: "{{ default_ethereum_client_images.lodestar }}" +lodestar_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +lodestar_container_volumes: + - "{{ lodestar_datadir }}:/data" + - "{{ lodestar_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +lodestar_container_command_extra_args: >- + {{ lodestar_container_command_extra_simple_args }} +lodestar_container_command_extra_simple_args: + - --paramsFile=/network-config/config.yaml + - --genesisStateFile=/network-config/genesis.ssz + - --ignoreWeakSubjectivityCheck + - --bootnodes={{ ethereum_cl_bootnodes | join(',') }} + - --rest.namespace="*" + - --network.connectToDiscv5Bootnodes + - --nat=true + - --chain.fastConfirmation=true +lodestar_validator_container_volumes: + - "{{ lodestar_validator_datadir }}:/validator-data" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +lodestar_validator_container_command_extra_args: + - --paramsFile=/network-config/config.yaml + - --graffiti={{ ansible_hostname }} +lodestar_validator_datadir: /data/lodestar-validator +lodestar_container_pull: true diff --git a/ansible/inventories/msf-2/group_vars/nethermind.yaml b/ansible/inventories/msf-2/group_vars/nethermind.yaml new file mode 100644 index 00000000..914197f2 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/nethermind.yaml @@ -0,0 +1,106 @@ +--- +# role: ethpandaops.general.bootstrap +nethermind_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/nethermind?format=txt') | select | list) + + nethermind_extra_github_users }} + +# role: ethpandaops.general.snapshot_fetcher (snapshot layout: mainnet/, FlatDb) +snapshot_fetcher_out_dir: /data/nethermind/nethermind_db + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: nethermind +# role: ethpandaops.general.nethermind +nethermind_container_name: execution +nethermind_container_image: "{{ default_ethereum_client_images.nethermind }}" +nethermind_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" +nethermind_container_entrypoint: + - /nethermind/nethermind +nethermind_container_volumes: + - "{{ nethermind_datadir }}:/data" + - "{{ nethermind_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +nethermind_container_command_extra_args: + # Restored jochemnet snapshot: the built-in mainnet config with our genesis.json as + # the chainspec (same genesis hash, amsterdamTime moved) and the db the snapshot + # ships. The preset turns on fast/snap sync; pin both off so nethermind only ever + # resumes from the restored db (nimbus-el is the only snap syncer on this network). + - --config=mainnet + - --Init.ChainSpecPath=/network-config/genesis.json + - --Init.BaseDbPath=nethermind_db/mainnet + - --FlatDb.Enabled=true + - --Sync.FastSync=false + - --Sync.SnapSync=false + - --JsonRpc.EnabledModules=Eth,Subscribe,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Admin + - --Discovery.Bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --Discovery.DiscoveryVersion=V5 + # chain id 1 otherwise adds the mainnet DNS tree and default discv5 bootnodes + - --Network.EnableEnrDiscovery=false + - --Discovery.UseDefaultDiscv5Bootnodes=false + - --Pruning.Mode=None + - --log=INFO + - --Seq.MinLevel=Info + - --Seq.ServerUrl={{ nethermind_seq_server }} + - --Seq.ApiKey={{ nethermind_seq_api_key }} + - --Metrics.PushGatewayUrl={{ nethermind_push_gateway }} +nethermind_container_pull: true + +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/metrics" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/nimbus.yaml b/ansible/inventories/msf-2/group_vars/nimbus.yaml new file mode 100644 index 00000000..11b4c7a8 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/nimbus.yaml @@ -0,0 +1,65 @@ +--- +# role: ethpandaops.general.bootstrap +nimbus_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/nimbus?format=txt') | select | list) + + nimbus_extra_github_users }} + +# role: validator_keys +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/nimbus-keys/" + dest: "{{ nimbus_validator_datadir }}/keys/" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/secrets/" + dest: "{{ nimbus_validator_datadir }}/secrets/" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ nimbus_validator_datadir }}/change_operations.json" + owner: nimbus + group: nimbus + +# role: ethpandaops.general.ethereum_node +# role: ethpandaops.general.validator_keys — nimbus runs validators in-process in +# the beacon container (no separate VC), so reshuffles stop/start the beacon. +validator_keys_container_name: beacon + +ethereum_node_cl: nimbus +ethereum_node_cl_checkpoint_sync_enabled: false +# Checkpointsync +nimbus_checkpoint_autoremove_enabled: true +nimbus_checkpoint_container_command_extra_args: + - --trusted-node-url={{ checkpoint_sync_url }} + - --network=/network-config + - --backfill=false +# role: ethpandaops.general.nimbus +nimbus_container_name: beacon +nimbus_container_image: "{{ default_ethereum_client_images.nimbus }}" +nimbus_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +nimbus_container_volumes: + - "{{ nimbus_datadir }}:/data" + - "{{ nimbus_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +nimbus_container_command_extra_args: >- + {{ nimbus_container_command_extra_simple_args + nimbus_container_command_extra_bootnode_args }} +nimbus_container_command_extra_simple_args: + - --network=/network-config + - --validator-monitor-auto=false + - --doppelganger-detection=off + - --log-level=DEBUG + - --light-client-data-serve=true + - --graffiti={{ ansible_hostname }} + # Gloas: the CL sets the payload gas target (PayloadAttributesV4.targetGasLimit) and + # nimbus v26.9.1 ignores GAS_LIMIT_SCHEDULE (default 60M), so its blocks pulled the gas + # limit back toward 60M after the fork while every other CL ramps toward 200M. + - --suggested-gas-limit=200000000 + +nimbus_container_command_extra_bootnode_args: >- + {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }} + +nimbus_validator_datadir: /data/nimbus-validator +nimbus_container_validator_volumes: + - "{{ nimbus_validator_datadir }}:/validator-data" +nimbus_container_pull: true diff --git a/ansible/inventories/msf-2/group_vars/nimbusel.yaml b/ansible/inventories/msf-2/group_vars/nimbusel.yaml new file mode 100644 index 00000000..fa9d4d96 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/nimbusel.yaml @@ -0,0 +1,99 @@ +--- +# role: ethpandaops.general.bootstrap +nimbusel_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/nimbus?format=txt') | select | list) + + nimbusel_extra_github_users }} + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: nimbusel +# role: ethpandaops.general.nimbusel +nimbusel_container_name: execution +nimbusel_container_image: "{{ default_ethereum_client_images.nimbusel }}" +nimbusel_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" + +nimbusel_container_volumes: + - "{{ nimbusel_datadir }}:/data" + - "{{ nimbusel_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +# Start these only after Gloas has finalized (playbook.yaml -l '!nimbusel' before): v0.4.2 +# snap sync needs a post-Amsterdam pivot (bals_download.nim:78) and the static enodes. +# No jochemnet nimbus-el snapshot: snap-sync from the restored ELs (experimental, hidden +# --debug-snap-sync in v0.4.2). Peer only with our snapshot ELs: real mainnet peers share +# our fork ID until amsterdamTime and would serve mainnet's state. Discovery off, +# one --static-peers per enode; p2p is also firewalled (scripts/shadowfork-ethrex-firewall.sh). +nimbusel_container_command_extra_args: >- + {{ nimbusel_shadowfork_args + + (shadowfork_el_static_enodes | map('regex_replace', '^', '--static-peers=') | list) }} +nimbusel_shadowfork_args: + - --network=/network-config/genesis.json + - --discv5=false + - --debug-snap-sync=true + # keep partial state across restarts (default: moved to a ~ backup, mainnet-sized) + - --debug-snap-sync-resume=true + - --rpc-api=admin,eth + - --ws-api=admin,eth + - --log-level=DEBUG + - --debug-parallel-state-root=true + - --debug-bal-state-prefetch=true + - --debug-bal-parallel-execution=true + +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/metrics" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/prysm.yaml b/ansible/inventories/msf-2/group_vars/prysm.yaml new file mode 100644 index 00000000..2f399d24 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/prysm.yaml @@ -0,0 +1,65 @@ +--- +# role: ethpandaops.general.bootstrap +prysm_extra_github_users: # handles not yet in the coredevs registry + - syjn99 +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/prysm?format=txt') | select | list) + + prysm_extra_github_users }} + +# role: validator_keys +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/prysm_wallet_pass.txt" + dest: "{{ prysm_validator_datadir }}/wallet_pass.txt" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/prysm/direct/accounts/all-accounts.keystore.json" + dest: "{{ prysm_validator_datadir }}/wallet/direct/accounts/all-accounts.keystore.json" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/prysm/keymanageropts.json" + dest: "{{ prysm_validator_datadir }}/wallet/direct/keymanageropts.json" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ prysm_validator_datadir }}/change_operations.json" + owner: prysm + group: prysm + +# role: ethpandaops.general.ethereum_node +ethereum_node_cl: prysm +prysm_container_tty: true +prysm_pprof_enabled: true +prysm_validator_container_tty: true +# role: ethpandaops.general.prysm +prysm_container_name: beacon +prysm_validator_container_name: validator +prysm_container_image: "{{ default_ethereum_client_images.prysm }}" +prysm_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +prysm_container_volumes: + - "{{ prysm_datadir }}:/data" + - "{{ prysm_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +prysm_container_entrypoint: + - /app/cmd/beacon-chain/beacon-chain +prysm_container_command_extra_args: >- + {{ prysm_container_command_extra_simple_args + prysm_container_command_extra_bootnode_args }} +prysm_container_command_extra_simple_args: + - --grpc-gateway-corsdomain=* + - --chain-config-file=/network-config/config.yaml + - --genesis-state=/network-config/genesis.ssz + - --contract-deployment-block={{ ethereum_network_deposit_contract_block }} + - --verbosity=DEBUG +prysm_container_command_extra_bootnode_args: >- + {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }} + +prysm_validator_container_image: "{{ default_ethereum_client_images.prysm_validator }}" +prysm_validator_container_volumes: + - "{{ prysm_validator_datadir }}:/validator-data" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +prysm_validator_container_entrypoint: + - /app/cmd/validator/validator +prysm_validator_container_command_extra_args: + - --chain-config-file=/network-config/config.yaml + # Hosts that set validator_display_name stamp it; the rest keep the hostname. + - --graffiti={{ validator_display_name | default(ansible_hostname) }} + - --enable-builder +prysm_validator_datadir: /data/prysm-validator diff --git a/ansible/inventories/msf-2/group_vars/reth.yaml b/ansible/inventories/msf-2/group_vars/reth.yaml new file mode 100644 index 00000000..33d109ea --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/reth.yaml @@ -0,0 +1,91 @@ +--- +# role: ethpandaops.general.bootstrap +reth_extra_github_users: [] # handles not yet in the coredevs registry +bootstrap_default_user_authorized_keys_github_team_el: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/reth?format=txt') | select | list) + + reth_extra_github_users }} + +# role: ethpandaops.general.snapshot_fetcher +snapshot_fetcher_out_dir: /data/reth +# v2 storage (snapshot.tar.zst is the old v1 format) +snapshot_fetcher_filename: snapshot-v2.tar.zst + +# role: ethpandaops.general.ethereum_node +ethereum_node_el: reth +# role: ethpandaops.general.reth +reth_container_name: execution +reth_container_image: "{{ default_ethereum_client_images.reth }}" +reth_container_env: + VIRTUAL_HOST: "{{ ethereum_node_rcp_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" + RUST_BACKTRACE: full +reth_container_volumes: + - "{{ reth_datadir }}:/data" + - "{{ reth_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" +reth_container_command_extra_args: + - --chain=/network-config/genesis.json + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --disable-discv4-discovery + # chain id 1 otherwise adds the mainnet DNS tree + - --disable-dns-discovery + - --http.api=rpc,eth,net,web3,admin,txpool + # devnet8 block gas limit is 200M; raise the RPC cap so eth_call/tracing can handle full-block txs + - --rpc.gascap=200000000 +prometheus_config: | + global: + scrape_interval: 30s + evaluation_interval: 30s + scrape_timeout: 10s + external_labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + ip_address: "{{ ansible_host }}" + network: "{{ ethereum_network_name }}" + testnet: "{{ ethereum_network_name }}" + execution_client: "{{ ethereum_node_el }}" + consensus_client: "{{ ethereum_node_cl }}" + supernode: "{{ ethereum_node_cl_supernode_enabled | bool | default(false) }}" + remote_write: + - queue_config: + batch_send_deadline: 5s + max_backoff: 500ms + max_samples_per_send: 500 + min_backoff: 50ms + max_shards: 100 + url: {{ prometheus_remote_push_url }} + remote_timeout: 10s + basic_auth: + username: {{ prometheus_remote_write_username }} + password: {{ prometheus_remote_write_password }} + scrape_configs: + - job_name: "prometheus" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "node" + metrics_path: "/metrics" + static_configs: + - targets: ["172.17.0.1:9100"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "exporter" + metrics_path: "/metrics" + static_configs: + - targets: ["ethereum-metrics-exporter:9090"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "consensus_node" + metrics_path: "/metrics" + static_configs: + - targets: ["{{ vars[ethereum_node_cl + '_container_name'] }}:{{ ethereum_node_cl_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" + - job_name: "execution" + metrics_path: "/metrics" + static_configs: + - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] + labels: + instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" diff --git a/ansible/inventories/msf-2/group_vars/teku.yaml b/ansible/inventories/msf-2/group_vars/teku.yaml new file mode 100644 index 00000000..748c4a63 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/teku.yaml @@ -0,0 +1,56 @@ +--- +# role: ethpandaops.general.bootstrap +teku_extra_github_users: # handles not yet in the coredevs registry + - siladu +bootstrap_default_user_authorized_keys_github_team_cl: >- + {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/teku?format=txt') | select | list) + + teku_extra_github_users }} + +# role: validator_keys +validator_keys_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/teku-keys/" + dest: "{{ teku_validator_datadir }}/keys/" + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/teku-secrets/" + dest: "{{ teku_validator_datadir }}/secrets/" + +validator_bls_sync_files: + - src: "{{ inventory_dir }}/files/validator_keys/{{ inventory_hostname }}/change_operations.json" + dest: "{{ teku_validator_datadir }}/change_operations.json" + owner: teku + group: teku +# role: ethpandaops.general.ethereum_node +# role: ethpandaops.general.validator_keys — teku runs validators in-process in +# the beacon container (no separate VC), so reshuffles stop/start the beacon. +validator_keys_container_name: beacon + +ethereum_node_cl: teku + +# role: ethpandaops.general.teku +teku_container_name: beacon +teku_container_image: "{{ default_ethereum_client_images.teku }}" +teku_container_env: + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" +teku_container_volumes: + - "{{ teku_datadir }}:/data" + - "{{ teku_auth_jwt_path }}:/execution-auth.jwt:ro" + - "{{ eth_testnet_config_dir }}:/network-config:ro" + +teku_container_command_extra_args : >- + {{ teku_container_command_extra_simple_args}} + + +teku_container_command_extra_simple_args: + - --network=/network-config/config.yaml + - --genesis-state=/network-config/genesis.ssz + - --ignore-weak-subjectivity-period-enabled + - --p2p-discovery-bootnodes={{ ethereum_cl_bootnodes | join(',') }} + - --validators-graffiti={{ ansible_hostname }} + - --logging=info + - --Xlog-include-p2p-warnings-enabled + - --metrics-block-timing-tracking-enabled + - --rest-api-docs-enabled +teku_validator_container_volumes: + - "{{ teku_validator_datadir }}:/validator-data" +teku_validator_datadir: /data/teku-validator diff --git a/ansible/inventories/msf-2/group_vars/tx_fuzz_blobs.yaml b/ansible/inventories/msf-2/group_vars/tx_fuzz_blobs.yaml new file mode 100644 index 00000000..c9d8ee77 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/tx_fuzz_blobs.yaml @@ -0,0 +1,13 @@ +--- +tx_fuzz_blobs_container_image: "{{ default_tooling_images.tx_fuzz }}" +tx_fuzz_blobs_container_name: tx_fuzz_blobs + +tx_fuzz_blobs_container_command: + - blobs + - --no-al + - --rpc=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_rpc_prefix }}{{ primary_bootnode + }}.{{ network_server_subdomain }} + - --sk={{ tx_fuzz_blobs_privkey }} + - --txcount=10 + +tx_fuzz_container_networks: "{{ docker_networks_shared }}" diff --git a/ansible/inventories/msf-2/group_vars/tx_fuzz_txs.yaml b/ansible/inventories/msf-2/group_vars/tx_fuzz_txs.yaml new file mode 100644 index 00000000..226680ff --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/tx_fuzz_txs.yaml @@ -0,0 +1,12 @@ +--- +tx_fuzz_container_image: "{{ default_tooling_images.tx_fuzz }}" +tx_fuzz_container_name: tx_fuzz_txs + +tx_fuzz_container_command: + - spam + - --no-al + - --rpc=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@{{ ethereum_node_rpc_prefix }}{{ primary_bootnode + }}.{{ network_server_subdomain }} + - --sk={{ tx_fuzz_txs_privkey }} + +tx_fuzz_container_networks: "{{ docker_networks_shared }}" diff --git a/ansible/inventories/msf-2/group_vars/xatu.yaml b/ansible/inventories/msf-2/group_vars/xatu.yaml new file mode 100644 index 00000000..a6365d00 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/xatu.yaml @@ -0,0 +1,91 @@ +--- +# role: ethpandaops.general.xatu_sentry +xatu_sentry_config_name: "{{ ethereum_network_name }}-{{ inventory_hostname }}" +xatu_sentry_config_server_address: "{{ ethereum_network_name }}-ingest.xatu.ethpandaops.io:443" # sops still holds devnet-8's ingest +xatu_sentry_config_server_auth_user: "{{ secret_xatu_sentry.user }}" +xatu_sentry_config_server_auth_password: "{{ secret_xatu_sentry.password }}" +xatu_sentry_config_network_name_override: "{{ ethereum_network_name }}" +xatu_sentry_container_networks: "{{ docker_networks_shared }}" +xatu_sentry_container_image: "{{ default_tooling_images.xatu_sentry }}" + +ethereum_node_xatu_sentry_enabled: true + +xatu_sentry_container_command: + - sentry + - --config=/config.yaml + +xatu_sentry_config: + logging: info + metricsAddr: :9090 + name: "{{ xatu_sentry_config_name }}" + ntpServer: time.google.com + ethereum: + beaconNodeAddress: http://{{ vars[ethereum_node_cl + '_container_name'] }}:{{ethereum_node_cl_ports_http_beacon}} + overrideNetworkName: "{{ xatu_sentry_config_network_name_override }}" + beaconSubscriptions: + - attestation + - block + - block_gossip + - chain_reorg + - finalized_checkpoint + - head + - voluntary_exit + - contribution_and_proof + - blob_sidecar + - data_column_sidecar + attestationData: + enabled: false + beaconCommittees: + enabled: true + proposerDuty: + enabled: true + forkChoice: + enabled: false + outputs: + - config: + address: "{{ xatu_sentry_config_server_address }}" + headers: + authorization: Basic {{ (xatu_sentry_config_server_auth_user + ':' + xatu_sentry_config_server_auth_password) | b64encode }} + maxExportBatchSize: 32 + maxQueueSize: 1000000 + workers: 5 + tls: true + filter: + eventNames: + - BEACON_API_ETH_V1_EVENTS_UNKNOWN + - BEACON_API_ETH_V1_BEACON_COMMITTEE + - BEACON_API_ETH_V1_EVENTS_BLOB_SIDECAR + - BEACON_API_ETH_V1_EVENTS_BLOCK + - BEACON_API_ETH_V1_EVENTS_BLOCK_V2 + - BEACON_API_ETH_V1_EVENTS_BLOCK_GOSSIP + - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG + - BEACON_API_ETH_V1_EVENTS_CHAIN_REORG_V2 + - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT + - BEACON_API_ETH_V1_EVENTS_FINALIZED_CHECKPOINT_V2 + - BEACON_API_ETH_V1_EVENTS_HEAD + - BEACON_API_ETH_V1_EVENTS_HEAD_V2 + - BEACON_API_ETH_V1_EVENTS_VOLUNTARY_EXIT + - BEACON_API_ETH_V1_EVENTS_VOLUNTARY_EXIT_V2 + - BEACON_API_ETH_V1_EVENTS_CONTRIBUTION_AND_PROOF + - BEACON_API_ETH_V1_EVENTS_CONTRIBUTION_AND_PROOF_V2 + - BEACON_API_ETH_V1_VALIDATOR_ATTESTATION_DATA + - MEMPOOL_TRANSACTION + - MEMPOOL_TRANSACTION_V2 + - BEACON_API_ETH_V2_BEACON_BLOCK + - BEACON_API_ETH_V2_BEACON_BLOCK_V2 + name: grpc-general + type: xatu + - config: + address: "{{ xatu_sentry_config_server_address }}" + headers: + authorization: Basic {{ (xatu_sentry_config_server_auth_user + ':' + xatu_sentry_config_server_auth_password) | b64encode }} + maxExportBatchSize: 128 + maxQueueSize: 1000000 + workers: 10 + tls: true + filter: + eventNames: + - BEACON_API_ETH_V1_EVENTS_ATTESTATION + - BEACON_API_ETH_V1_EVENTS_ATTESTATION_V2 + name: grpc-attestation + type: xatu diff --git a/ansible/inventories/msf-2/group_vars/xatu_sentry.yaml b/ansible/inventories/msf-2/group_vars/xatu_sentry.yaml new file mode 100644 index 00000000..f49b32b1 --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/xatu_sentry.yaml @@ -0,0 +1,2 @@ +--- +ethereum_node_xatu_sentry_enabled: true diff --git a/ansible/inventories/msf-2/host_vars/buildoor-teku-nethermind-1.yaml b/ansible/inventories/msf-2/host_vars/buildoor-teku-nethermind-1.yaml new file mode 100644 index 00000000..cad4fa0c --- /dev/null +++ b/ansible/inventories/msf-2/host_vars/buildoor-teku-nethermind-1.yaml @@ -0,0 +1,4 @@ +--- +# Own wallet (genesis mnemonic index 21): on sepsf-1 both buildoor instances sent from +# mnemonic-12 and their deposit/bid txs collided on nonces. +buildoor_wallet_privkey: "{{ secret_buildoor_wallet_privkey_2 }}" diff --git a/ansible/inventories/msf-2/host_vars/localhost b/ansible/inventories/msf-2/host_vars/localhost new file mode 100644 index 00000000..e187c8f2 --- /dev/null +++ b/ansible/inventories/msf-2/host_vars/localhost @@ -0,0 +1,2 @@ +ansible_connection: local +ansible_python_interpreter: "{{ ansible_playbook_python }}" diff --git a/terraform/msf-2/ansible_inventory.tmpl b/terraform/msf-2/ansible_inventory.tmpl new file mode 100644 index 00000000..7a2829de --- /dev/null +++ b/terraform/msf-2/ansible_inventory.tmpl @@ -0,0 +1,122 @@ +localhost + +[all:vars] +ethereum_network_name=${ethereum_network_name} + +%{ for gid, group in groups ~} +[${replace(gid, "-", "_")}] +%{ for key, host in hosts ~} +%{ if host.group == gid ~} +${host.hostname} ansible_host=${host.ip} ipv6=${host.ipv6} cloud=${host.cloud} cloud_region=${host.region} arch=${host.arch} ethereum_node_cl_supernode_enabled=${title(host.supernode)} %{ if tonumber(host.validator_end) > 0 }validator_start=${host.validator_start} validator_end=${host.validator_end}%{ endif }%{ if host.builder_index != "" } builder_index=${host.builder_index}%{ endif } +%{ endif ~} +%{ endfor ~} +%{ if gid == "lighthouse-reth" ~} +%{ for key, host in hosts ~} +%{ if host.group == "mev-relay" ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} +%{ endif ~} + +%{ endfor ~} + +%{ if !contains(keys(groups), "lighthouse-reth") ~} +[lighthouse_reth] +%{ for key, host in hosts ~} +%{ if host.group == "mev-relay" ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} +%{ endif ~} + +# Consensus client groups + +%{ for cl in ["lighthouse", "lodestar", "nimbus", "teku", "prysm", "grandine"] ~} +[${cl}:children] +%{ for gid, group in groups ~} +%{ if split("-", gid)[0] == "${cl}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ if split("-", gid)[0] == "buildoor" && length(split("-", gid)) >= 2 ~} +%{ if split("-", gid)[1] == "${cl}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endif ~} +%{ endfor ~} +%{ if cl == "lighthouse" && contains(keys(groups), "mev-relay") ~} +mev_relay +%{ endif ~} +%{ endfor ~} + +# Execution client groups + +%{ for el in ["besu", "ethereumjs", "geth", "nethermind", "erigon", "reth", "nimbusel", "ethrex"] ~} +[${el}:children] +%{ for gid, group in groups ~} +%{ if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" && split("-", gid)[0] != "buildoor" ~} +%{ if length(split("-", gid)) >= 2 && split("-", gid)[1] == "${el}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endif ~} +%{ if split("-", gid)[0] == "buildoor" && length(split("-", gid)) >= 3 ~} +%{ if split("-", gid)[2] == "${el}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endif ~} +%{ endfor ~} +%{ if el == "reth" && contains(keys(groups), "mev-relay") ~} +mev_relay +%{ endif ~} +%{ endfor ~} + +# Buildoor host aggregator + +[buildoor:children] +%{ for gid, group in groups ~} +%{ if split("-", gid)[0] == "buildoor" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endfor ~} + +# ansible_group_priority MUST live in the inventory (not group_vars/) — it is consulted +# while loading group_vars, so a value placed in group_vars/buildoor.yaml is read too late +# and the buildoor overrides lose to alphabetically-later sibling groups (geth, prysm). +[buildoor:vars] +ansible_group_priority=100 + +# Global groups + +[consensus_node:children] +%{ for x,y in merge( { for gid, group in groups : split("-", gid)[0] => true... if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" && split("-", gid)[0] != "buildoor" } ) ~} +${x} +%{ endfor ~} +%{ if length([for gid, group in groups : gid if split("-", gid)[0] == "buildoor"]) > 0 ~} +buildoor +%{ endif ~} + +[execution_node:children] +%{ for x,y in merge( { for gid, group in groups : split("-", gid)[1] => true... if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" && split("-", gid)[0] != "buildoor" && length(split("-", gid)) >= 2 } ) ~} +${x} +%{ endfor ~} +%{ if length([for gid, group in groups : gid if split("-", gid)[0] == "buildoor"]) > 0 ~} +buildoor +%{ endif ~} + +[ethereum_node:children] +consensus_node +execution_node + +%{ if contains(keys(groups), "bootnode") ~} +[dns_server:children] +bootnode +%{ endif ~} + +[mev_boost:children] +consensus_node + +[arm] +%{ for key, host in hosts ~} +%{ if can(regex("arm", key)) ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} diff --git a/terraform/msf-2/cloudflare.tf b/terraform/msf-2/cloudflare.tf new file mode 100644 index 00000000..67bd8103 --- /dev/null +++ b/terraform/msf-2/cloudflare.tf @@ -0,0 +1,60 @@ + +//////////////////////////////////////////////////////////////////////////////////////// +// DNS NAMES +//////////////////////////////////////////////////////////////////////////////////////// + +data "cloudflare_zone" "default" { + name = "ethpandaops.io" +} + +locals { + # Combine bootnodes from both providers + bootnodes = merge( + { + for vm in local.digitalocean_vms : vm.id => { + name = vm.name + has_ipv6 = vm.ipv6 + ipv4 = digitalocean_droplet.main[vm.id].ipv4_address + ipv6 = try(digitalocean_droplet.main[vm.id].ipv6_address, null) + } if can(regex("bootnode", vm.name)) + }, + { + for vm in local.hcloud_vms : vm.id => { + name = vm.name + has_ipv6 = vm.ipv6_enabled + ipv4 = hcloud_server.main[vm.id].ipv4_address + ipv6 = try(hcloud_server.main[vm.id].ipv6_address, null) + } if can(regex("bootnode", vm.name)) + } + ) +} + +resource "cloudflare_record" "server_record_v4" { + for_each = local.bootnodes + zone_id = data.cloudflare_zone.default.id + name = "${each.value.name}.${var.ethereum_network}" + type = "A" + value = each.value.ipv4 + proxied = false + ttl = 120 +} + +resource "cloudflare_record" "server_record_v6" { + for_each = { for k, v in local.bootnodes : k => v if v.has_ipv6 } + zone_id = data.cloudflare_zone.default.id + name = "${each.value.name}.${var.ethereum_network}" + type = "AAAA" + value = each.value.ipv6 + proxied = false + ttl = 120 +} + +resource "cloudflare_record" "server_record_ns" { + for_each = local.bootnodes + zone_id = data.cloudflare_zone.default.id + name = "srv.${var.ethereum_network}" + type = "NS" + value = "${each.value.name}.${var.ethereum_network}.${data.cloudflare_zone.default.name}" + proxied = false + ttl = 120 +} diff --git a/terraform/msf-2/digitalocean.tf b/terraform/msf-2/digitalocean.tf new file mode 100644 index 00000000..0eaf4ff6 --- /dev/null +++ b/terraform/msf-2/digitalocean.tf @@ -0,0 +1,181 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "digitalocean_project_name" { + type = string + default = "glamsterdam-devnets" +} + +variable "digitalocean_ssh_key_name" { + type = string + default = "shared-devops-eth2" +} + +variable "digitalocean_supernode_size" { + type = string + default = "so1_5-8vcpu-64gb-intel" # 1.8 TB disk (DO's largest) for the jochemnet snapshots +} + +variable "digitalocean_fullnode_size" { + type = string + default = "s-8vcpu-16gb" +} + +variable "digitalocean_regions" { + default = [ + "nyc1", + "sgp1", + "lon1", + "nyc3", + "ams3", + "fra1", + "tor1", + "blr1", + "sfo3" # syd1 dropped: no so1_5-8vcpu-64gb-intel there + ] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + digitalocean_vpcs = { + for region in var.digitalocean_regions : region => { + name = "${var.ethereum_network}-${region}" + region = region + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.digitalocean_regions, region)) + } + } +} + +locals { + digitalocean_vm_groups = flatten([ + for node in local.digitalocean_nodes : [ + for i in range(0, node.count) : { + group_name = node.name + id = "${node.name}-${node.start_index + i + 1}" + vms = { + "${i + 1}" = { + # Validator range for this instance + val_start = node.validator_start + (i * (node.validator_end - node.validator_start) / node.count) + val_end = min( + node.validator_start + ((i + 1) * (node.validator_end - node.validator_start) / node.count), + node.validator_end + ) + validator_count = node.count > 0 ? (node.validator_end - node.validator_start) / node.count : 0 + + # Builder index for buildoor nodes (start index + instance offset); null = no tag + builder_index = node.builder_start != null ? node.builder_start + i : null + + # Supernode: explicit > bootnode/mev > validator_count >= 128 + supernode = ( + node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) + + # Size: explicit > supernode-based default + size = ( + node.size != null ? node.size : + (node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) ? var.digitalocean_supernode_size : var.digitalocean_fullnode_size + ) + + region = node.region != null ? node.region : var.digitalocean_regions[ + parseint(substr(md5("${node.name}-${node.start_index + i + 1}"), 0, 8), 16) % length(var.digitalocean_regions) + ] + ipv6 = node.ipv6 + arch = "amd64" + } + } + } + ] + ]) +} + +locals { + digitalocean_default_region = "ams3" + digitalocean_default_size = var.digitalocean_fullnode_size + digitalocean_default_image = "debian-13-x64" + digitalocean_global_tags = [ + "Owner:Devops", + "EthNetwork:${var.ethereum_network}" + ] + + digitalocean_vms = flatten([ + for group in local.digitalocean_vm_groups : [ + for vm_key, vm in group.vms : { + id = group.id + group_key = group.group_name + vm_key = vm_key + + name = group.id + ssh_keys = [data.digitalocean_ssh_key.main.fingerprint] + region = vm.region + image = local.digitalocean_default_image + size = vm.size + resize_disk = true + monitoring = true + backups = false + ipv6 = vm.ipv6 + vpc_uuid = digitalocean_vpc.main[vm.region].id + + tags = concat(local.digitalocean_global_tags, [ + "group_name:${group.group_name}", + "val_start:${vm.val_start}", + "val_end:${vm.val_end}", + "supernode:${vm.supernode ? "True" : "False"}", + "arch:${vm.arch}", + ], compact([ + can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, + can(regex("mev-relay", group.group_name)) ? "mev-relay:${var.ethereum_network}" : null, + vm.builder_index != null ? "builder_index:${vm.builder_index}" : null + ])) + } + ] + ]) +} + +//////////////////////////////////////////////////////////////////////////////////////// +// DIGITALOCEAN RESOURCES +//////////////////////////////////////////////////////////////////////////////////////// +data "digitalocean_project" "main" { + name = var.digitalocean_project_name +} + +data "digitalocean_ssh_key" "main" { + name = var.digitalocean_ssh_key_name +} + +resource "digitalocean_vpc" "main" { + for_each = local.digitalocean_vpcs + + name = each.value["name"] + region = each.value["region"] + ip_range = each.value["ip_range"] +} + +resource "digitalocean_droplet" "main" { + for_each = { + for vm in local.digitalocean_vms : vm.id => vm + } + name = "${var.ethereum_network}-${each.value.name}" + region = each.value.region + ssh_keys = each.value.ssh_keys + image = each.value.image + size = each.value.size + resize_disk = each.value.resize_disk + monitoring = each.value.monitoring + backups = each.value.backups + ipv6 = each.value.ipv6 + vpc_uuid = each.value.vpc_uuid + tags = each.value.tags +} + +resource "digitalocean_project_resources" "droplets" { + for_each = digitalocean_droplet.main + project = data.digitalocean_project.main.id + resources = [each.value.urn] +} diff --git a/terraform/msf-2/firewall.tf b/terraform/msf-2/firewall.tf new file mode 100644 index 00000000..9e3db887 --- /dev/null +++ b/terraform/msf-2/firewall.tf @@ -0,0 +1,370 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// DIGITALOCEAN FIREWALLS +//////////////////////////////////////////////////////////////////////////////////////// +variable "el_p2p_extra_source_addresses" { + description = "Extra CIDRs allowed to reach EL p2p 30303 (e.g. a local join host)" + type = list(string) + default = [] +} + +resource "digitalocean_firewall" "main" { + count = length(local.digitalocean_vms) > 0 ? 1 : 0 + name = "${var.ethereum_network}-nodes" + tags = [ + "EthNetwork:${var.ethereum_network}" + ] + + // SSH + inbound_rule { + protocol = "tcp" + port_range = "22" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Allow all inbound ICMP + inbound_rule { + protocol = "icmp" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Nginx / Web + inbound_rule { + protocol = "tcp" + port_range = "80" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "443" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Consensus layer p2p port + inbound_rule { + protocol = "tcp" + port_range = "9000-9001" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "9000-9002" + source_addresses = ["0.0.0.0/0", "::/0"] + } + // Prysm QUIC port (default p2p-quic-port=13000) + inbound_rule { + protocol = "udp" + port_range = "13000" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Execution layer p2p port: our droplets only. Chain id 1 + mainnet's fork id until + // amsterdamTime, so real mainnet peers would otherwise connect. + inbound_rule { + protocol = "tcp" + port_range = "30303" + source_tags = ["EthNetwork:${var.ethereum_network}"] + source_addresses = length(var.el_p2p_extra_source_addresses) > 0 ? var.el_p2p_extra_source_addresses : null + } + inbound_rule { + protocol = "udp" + port_range = "30303" + source_tags = ["EthNetwork:${var.ethereum_network}"] + source_addresses = length(var.el_p2p_extra_source_addresses) > 0 ? var.el_p2p_extra_source_addresses : null + } + inbound_rule { + protocol = "tcp" + port_range = "42069" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "42069" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Engine rpc-snooper api + inbound_rule { + protocol = "tcp" + port_range = "8961" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // TYSM control API (bad-tysm reaches back on this port) + inbound_rule { + protocol = "tcp" + port_range = "8675" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Allow all outbound traffic + outbound_rule { + protocol = "tcp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + outbound_rule { + protocol = "udp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + outbound_rule { + protocol = "icmp" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + depends_on = [digitalocean_project_resources.droplets] +} + +resource "digitalocean_firewall" "bootnode" { + count = contains(keys(digitalocean_droplet.main), "bootnode-1") ? 1 : 0 + name = "${var.ethereum_network}-nodes-bootnode" + tags = [ + "bootnode:${var.ethereum_network}" + ] + + // DNS + inbound_rule { + protocol = "tcp" + port_range = "53" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "53" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Bootnodoor P2P + inbound_rule { + protocol = "tcp" + port_range = "9010" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "9010" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + depends_on = [digitalocean_project_resources.droplets] +} + +resource "digitalocean_firewall" "mev_relay" { + count = contains(keys(digitalocean_droplet.main), "mev-relay-1") ? 1 : 0 + name = "${var.ethereum_network}-nodes-mev-relay" + tags = ["mev-relay:${var.ethereum_network}"] + + // mev-relay ports + inbound_rule { + protocol = "tcp" + port_range = "9060-9062" + source_addresses = ["0.0.0.0/0", "::/0"] + } + depends_on = [digitalocean_project_resources.droplets] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// HETZNER FIREWALLS +//////////////////////////////////////////////////////////////////////////////////////// +resource "hcloud_firewall" "machine_firewall" { + count = local.hetzner_has_servers ? 1 : 0 + name = "${var.ethereum_network}-firewall" + + apply_to { + label_selector = "EthNetwork=${var.ethereum_network}" + } + + // SSH + rule { + description = "Allow SSH" + direction = "in" + protocol = "tcp" + port = "22" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Allow all inbound ICMP + rule { + description = "Allow all inbound ICMP" + direction = "in" + protocol = "icmp" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Nginx / Web + rule { + description = "Allow HTTP" + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow HTTPS" + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Consensus layer p2p port + rule { + description = "Allow consensus p2p port TCP" + direction = "in" + protocol = "tcp" + port = "9000-9002" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow consensus p2p port UDP" + direction = "in" + protocol = "udp" + port = "9000-9002" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Prysm QUIC port (default p2p-quic-port=13000) + rule { + description = "Allow Prysm QUIC port UDP" + direction = "in" + protocol = "udp" + port = "13000" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Execution layer p2p Port + rule { + description = "Allow execution p2p port TCP" + direction = "in" + protocol = "tcp" + port = "30303" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution p2p port UDP" + direction = "in" + protocol = "udp" + port = "30303" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution torrent port TCP" + direction = "in" + protocol = "tcp" + port = "42069" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution torrent port UDP" + direction = "in" + protocol = "udp" + port = "42069" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Engine rpc-snooper api + rule { + description = "Allow engine snooper api port TCP" + direction = "in" + protocol = "tcp" + port = "8961" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // TYSM control API (bad-tysm reaches back on this port) + rule { + description = "Allow TYSM control API port TCP" + direction = "in" + protocol = "tcp" + port = "8675" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Allow all outbound traffic + rule { + description = "Allow all outbound traffic TCP" + direction = "out" + protocol = "tcp" + port = "1-65535" + destination_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow all outbound traffic UDP" + direction = "out" + protocol = "udp" + port = "1-65535" + destination_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow all outbound traffic ICMP" + direction = "out" + protocol = "icmp" + destination_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_firewall" "bootnode_firewall" { + count = contains(keys(hcloud_server.main), "bootnode-1") ? 1 : 0 + name = "${var.ethereum_network}-bootnode-firewall" + + apply_to { + label_selector = "bootnode=${var.ethereum_network}" + } + + // DNS + rule { + description = "Allow DNS UDP" + direction = "in" + protocol = "udp" + port = "53" + source_ips = ["0.0.0.0/0", "::/0"] + } + rule { + description = "Allow DNS TCP" + direction = "in" + protocol = "tcp" + port = "53" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Bootnodoor P2P + rule { + description = "Allow Bootnodoor P2P port TCP" + direction = "in" + protocol = "tcp" + port = "9010" + source_ips = ["0.0.0.0/0", "::/0"] + } + rule { + description = "Allow Bootnodoor P2P port UDP" + direction = "in" + protocol = "udp" + port = "9010" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_firewall" "mev_relay_firewall" { + count = contains(keys(hcloud_server.main), "mev-relay-1") ? 1 : 0 + name = "${var.ethereum_network}-mev-relay-firewall" + + apply_to { + label_selector = "mev=${var.ethereum_network}" + } + + // mev-relay ports + rule { + description = "Allow MEV Relay ports" + direction = "in" + protocol = "tcp" + port = "9060-9062" + source_ips = ["0.0.0.0/0", "::/0"] + } +} diff --git a/terraform/msf-2/hetzner.tf b/terraform/msf-2/hetzner.tf new file mode 100644 index 00000000..9b7366a0 --- /dev/null +++ b/terraform/msf-2/hetzner.tf @@ -0,0 +1,220 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "hcloud_ssh_key_fingerprint" { + type = string + default = "d6:76:2d:9c:5b:33:80:ff:0f:09:a2:10:9b:58:7e:dc" +} + +variable "hetzner_supernode_size" { + type = string + default = "cpx62" +} + +variable "hetzner_fullnode_size" { + type = string + default = "cpx42" +} + +variable "hetzner_regions" { + default = [ + "fsn1", + "nbg1", + "hel1", + ] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// DATA SOURCES +//////////////////////////////////////////////////////////////////////////////////////// +data "hcloud_server_type" "supernode" { + count = local.hetzner_has_servers ? 1 : 0 + name = var.hetzner_supernode_size +} + +data "hcloud_server_type" "fullnode" { + count = local.hetzner_has_servers ? 1 : 0 + name = var.hetzner_fullnode_size +} + +data "hcloud_datacenters" "all" { + count = local.hetzner_has_servers ? 1 : 0 +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + hetzner_has_servers = length(local.hetzner_nodes) > 0 + + required_hetzner_server_type_ids = local.hetzner_has_servers ? toset([ + tostring(data.hcloud_server_type.supernode[0].id), + tostring(data.hcloud_server_type.fullnode[0].id), + ]) : toset([]) + + hetzner_available_locations = local.hetzner_has_servers ? distinct([ + for dc in data.hcloud_datacenters.all[0].datacenters : dc.location.name + if alltrue([for tid in local.required_hetzner_server_type_ids : contains([for id in dc.available_server_type_ids : tostring(id)], tid)]) + ]) : [] + + hetzner_regions_filtered = [ + for r in var.hetzner_regions : r if contains(local.hetzner_available_locations, r) + ] + + hetzner_regions_effective = length(local.hetzner_regions_filtered) > 0 ? local.hetzner_regions_filtered : var.hetzner_regions + + hetzner_network = { + for region in var.hetzner_regions : region => { + name = "${var.ethereum_network}-${region}" + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.hetzner_regions, region)) + } + } + hetzner_network_subnets = { + for region in var.hetzner_regions : region => { + zone = "eu-central" + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.hetzner_regions, region)) + } + } +} + +locals { + hetzner_vm_groups = flatten([ + for node in local.hetzner_nodes : [ + for i in range(0, node.count) : { + group_name = node.name + id = "${node.name}-${node.start_index + i + 1}" + vms = { + "${i + 1}" = { + # Validator range for this instance + val_start = node.validator_start + (i * (node.validator_end - node.validator_start) / node.count) + val_end = min( + node.validator_start + ((i + 1) * (node.validator_end - node.validator_start) / node.count), + node.validator_end + ) + validator_count = node.count > 0 ? (node.validator_end - node.validator_start) / node.count : 0 + + # Builder index for buildoor nodes (start index + instance offset); null = no tag + builder_index = node.builder_start != null ? node.builder_start + i : null + + # Supernode: explicit > bootnode/mev > validator_count >= 128 + supernode = ( + node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) + + # Size: explicit > supernode-based default + size = ( + node.size != null ? node.size : + (node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) ? var.hetzner_supernode_size : var.hetzner_fullnode_size + ) + + location = node.location != null ? node.location : local.hetzner_regions_effective[ + parseint(substr(md5("${node.name}-${node.start_index + i + 1}"), 0, 8), 16) % length(local.hetzner_regions_effective) + ] + ipv4_enabled = node.ipv4_enabled + ipv6_enabled = node.ipv6_enabled + } + } + } + ] + ]) +} + +locals { + hcloud_default_location = "nbg1" + hcloud_default_image = "debian-13" + hcloud_default_server_type = var.hetzner_fullnode_size + hcloud_global_labels = [ + "Owner:Devops", + "EthNetwork:${var.ethereum_network}" + ] + + hcloud_vms = flatten([ + for group in local.hetzner_vm_groups : [ + for vm_key, vm in group.vms : { + id = group.id + group_key = group.group_name + vm_key = vm_key + + name = group.id + ipv4_enabled = vm.ipv4_enabled + ipv6_enabled = vm.ipv6_enabled + ssh_keys = local.hetzner_has_servers ? [data.hcloud_ssh_key.main[0].id] : [] + location = vm.location + image = local.hcloud_default_image + server_type = vm.size + backups = false + + # Architecture: cax* = ARM64, everything else = AMD64 + arch = can(regex("^cax", vm.size)) ? "arm64" : "amd64" + + labels = concat(local.hcloud_global_labels, [ + "group_name:${group.group_name}", + "val_start:${vm.val_start}", + "val_end:${vm.val_end}", + "supernode:${vm.supernode ? "True" : "False"}", + "arch:${can(regex("^cax", vm.size)) ? "arm64" : "amd64"}", + ], compact([ + can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, + can(regex("mev-relay", group.group_name)) ? "mev:${var.ethereum_network}" : null, + vm.builder_index != null ? "builder_index:${vm.builder_index}" : null + ])) + } + ] + ]) +} + +//////////////////////////////////////////////////////////////////////////////////////// +// HETZNER RESOURCES +//////////////////////////////////////////////////////////////////////////////////////// +resource "hcloud_network" "main" { + for_each = local.hetzner_has_servers ? local.hetzner_network : {} + name = each.value.name + ip_range = each.value.ip_range +} + +resource "hcloud_network_subnet" "main" { + for_each = local.hetzner_has_servers ? local.hetzner_network_subnets : {} + network_id = hcloud_network.main[each.key].id + type = "cloud" + network_zone = each.value.zone + ip_range = each.value.ip_range +} + +data "hcloud_ssh_key" "main" { + count = local.hetzner_has_servers ? 1 : 0 + fingerprint = var.hcloud_ssh_key_fingerprint +} + +resource "hcloud_server" "main" { + for_each = { + for vm in local.hcloud_vms : vm.id => vm + } + name = "${var.ethereum_network}-${each.value.name}" + image = each.value.image + server_type = each.value.server_type + location = each.value.location + ssh_keys = each.value.ssh_keys + backups = each.value.backups + labels = { for label in each.value.labels : split(":", label)[0] => split(":", label)[1] } + public_net { + ipv4_enabled = each.value.ipv4_enabled + ipv6_enabled = each.value.ipv6_enabled + } + + lifecycle { + ignore_changes = [location] + } +} + +resource "hcloud_server_network" "main" { + for_each = { + for vm in local.hcloud_vms : vm.id => vm + } + server_id = hcloud_server.main[each.key].id + network_id = hcloud_network.main[hcloud_server.main[each.key].location].id +} diff --git a/terraform/msf-2/main.tf b/terraform/msf-2/main.tf new file mode 100644 index 00000000..5bb11651 --- /dev/null +++ b/terraform/msf-2/main.tf @@ -0,0 +1,99 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// TERRAFORM PROVIDERS & BACKEND +//////////////////////////////////////////////////////////////////////////////////////// +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.28" + } + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 3.0" + } + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.42.1" + } + } +} + +terraform { + backend "s3" { + skip_credentials_validation = true + skip_metadata_api_check = true + endpoints = { s3 = "https://fra1.digitaloceanspaces.com" } + skip_requesting_account_id = true + skip_s3_checksum = true + region = "us-east-1" + bucket = "merge-testnets" + key = "infrastructure/msf-2/terraform.tfstate" + } +} + +provider "digitalocean" { + http_retry_max = 20 +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +provider "hcloud" { + token = var.glamsterdam_hcloud_token +} + +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "cloudflare_api_token" { + type = string + sensitive = true + description = "Cloudflare API Token" +} + +variable "glamsterdam_hcloud_token" { + type = string + sensitive = true + default = "" + description = "Hetzner Cloud API Token (optional if not using Hetzner)" +} + +variable "ethereum_network" { + type = string + default = "glamsterdam-msf-2" +} + +variable "base_cidr_block" { + default = "10.33.0.0/16" # sepsf-1/2 use 10.30/10.31, msf-1 10.32: VPC ranges must not overlap per region +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + # Normalize node entries with defaults and calculate starting index for continuous numbering + nodes_normalized = [ + for idx, node in var.nodes : { + name = node.name + count = node.count + cloud = node.cloud + validator_start = try(node.validator_start, 0) + validator_end = try(node.validator_end, 0) + builder_start = try(node.builder_start, null) + size = try(node.size, null) + region = try(node.region, null) + location = try(node.location, try(node.region, null)) + supernode = try(node.supernode, null) + ipv6 = try(node.ipv6, true) + ipv4_enabled = try(node.ipv4_enabled, true) + ipv6_enabled = try(node.ipv6_enabled, true) + # Calculate starting index: sum of counts from all previous entries with same name + start_index = sum(concat([for i, n in var.nodes : n.count if i < idx && n.name == node.name], [0])) + } + ] + + # Filter by cloud provider (only nodes with count > 0) + digitalocean_nodes = [for n in local.nodes_normalized : n if n.cloud == "digitalocean" && n.count > 0] + hetzner_nodes = [for n in local.nodes_normalized : n if n.cloud == "hetzner" && n.count > 0] +} diff --git a/terraform/msf-2/nodes.tf b/terraform/msf-2/nodes.tf new file mode 100644 index 00000000..e908c577 --- /dev/null +++ b/terraform/msf-2/nodes.tf @@ -0,0 +1,113 @@ +######################################################################################## +# NODE DEFINITIONS +# +# Define your fleet as a list of node entries. Each entry supports: +# +# Required: +# - name : Node type (e.g., "lighthouse-geth-super", "bootnode") +# - count : Number of instances +# - cloud : "digitalocean" or "hetzner" +# +# Optional: +# - validator_start : First validator index (default: 0) +# - validator_end : Last validator index (default: 0) +# - size : Instance size override (provider-specific) +# - region : Region override (digitalocean) or location (hetzner) +# - supernode : Force supernode=true/false (auto-detected from name) +# - builder_start : First builder index (buildoor nodes only). Exposes a +# builder_index=N server tag and inventory var per instance. +# +# Examples: +# { name = "bootnode", count = 1, cloud = "digitalocean" } +# { name = "lighthouse-geth-super", count = 2, cloud = "hetzner", validator_start = 0, validator_end = 200 } +# { name = "mev-relay", count = 1, cloud = "hetzner", size = "ccx53" } +# +######################################################################################## + +variable "nodes" { + description = "List of node definitions for the devnet" + default = [ + # Every EL except nimbus-el restores a jochemnet snapshot (up to ~1.5 TB on disk), + # so every node with an EL gets the 1.8 TB storage-optimized size. + { name = "bootnode", count = 1, cloud = "digitalocean" }, + { name = "buildoor-lighthouse-geth", count = 1, cloud = "digitalocean", size = "so1_5-8vcpu-64gb-intel", builder_start = 0 }, + { name = "buildoor-teku-nethermind", count = 1, cloud = "digitalocean", size = "so1_5-8vcpu-64gb-intel", builder_start = 1 }, + + # Full 6 CL x 6 EL matrix, every pair once: 36 x 1000 keys over [0,36000) + # (NUMBER_OF_VALIDATORS=36000). Nimbus-el (no validators) below the ethrex nodes. + + # Geth + { name = "lighthouse-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 0, validator_end = 1000 }, + { name = "prysm-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 1000, validator_end = 2000 }, + { name = "teku-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 2000, validator_end = 3000 }, + { name = "nimbus-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 3000, validator_end = 4000 }, + { name = "lodestar-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 4000, validator_end = 5000 }, + { name = "grandine-geth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 5000, validator_end = 6000 }, + + # Nethermind + { name = "lighthouse-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 6000, validator_end = 7000 }, + { name = "prysm-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 7000, validator_end = 8000 }, + { name = "teku-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 8000, validator_end = 9000 }, + { name = "nimbus-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 9000, validator_end = 10000 }, + { name = "lodestar-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 10000, validator_end = 11000 }, + { name = "grandine-nethermind", count = 1, cloud = "digitalocean", supernode = true, validator_start = 11000, validator_end = 12000 }, + + # Besu + { name = "lighthouse-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 12000, validator_end = 13000 }, + { name = "prysm-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 13000, validator_end = 14000 }, + { name = "teku-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 14000, validator_end = 15000 }, + { name = "nimbus-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 15000, validator_end = 16000 }, + { name = "lodestar-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 16000, validator_end = 17000 }, + { name = "grandine-besu", count = 1, cloud = "digitalocean", supernode = true, validator_start = 17000, validator_end = 18000 }, + + # Reth + { name = "lighthouse-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 18000, validator_end = 19000 }, + { name = "prysm-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 19000, validator_end = 20000 }, + { name = "teku-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 20000, validator_end = 21000 }, + { name = "nimbus-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 21000, validator_end = 22000 }, + { name = "lodestar-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 22000, validator_end = 23000 }, + { name = "grandine-reth", count = 1, cloud = "digitalocean", supernode = true, validator_start = 23000, validator_end = 24000 }, + + # Erigon + { name = "lighthouse-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 24000, validator_end = 25000 }, + { name = "prysm-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 25000, validator_end = 26000 }, + { name = "teku-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 26000, validator_end = 27000 }, + { name = "nimbus-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 27000, validator_end = 28000 }, + { name = "lodestar-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 28000, validator_end = 29000 }, + { name = "grandine-erigon", count = 1, cloud = "digitalocean", supernode = true, validator_start = 29000, validator_end = 30000 }, + + # Ethrex + { name = "lighthouse-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 30000, validator_end = 31000 }, + { name = "prysm-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 31000, validator_end = 32000 }, + { name = "teku-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 32000, validator_end = 33000 }, + { name = "nimbus-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 33000, validator_end = 34000 }, + { name = "lodestar-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 34000, validator_end = 35000 }, + { name = "grandine-ethrex", count = 1, cloud = "digitalocean", supernode = true, validator_start = 35000, validator_end = 36000 }, + + # Nimbus-el: no snapshot, snap-syncs from our ELs. No validators. + { name = "lighthouse-nimbusel", count = 1, cloud = "digitalocean", size = "so1_5-8vcpu-64gb-intel", region = "fra1" }, + { name = "nimbus-nimbusel", count = 1, cloud = "digitalocean", size = "so1_5-8vcpu-64gb-intel", region = "lon1" }, + ] + + validation { + condition = alltrue([ + for n in var.nodes : + try(n.validator_start, 0) >= 0 && try(n.validator_start, 0) <= try(n.validator_end, 0) + ]) + error_message = "Each node must satisfy 0 <= validator_start <= validator_end. Omit both fields (or set both to 0) for nodes without validators." + } + + validation { + condition = alltrue(flatten([ + for i, a in var.nodes : [ + for j, b in var.nodes : + i >= j || + try(a.validator_end, 0) == 0 || + try(b.validator_end, 0) == 0 || + try(a.validator_end, 0) <= try(b.validator_start, 0) || + try(b.validator_end, 0) <= try(a.validator_start, 0) + ] + ])) + error_message = "Validator ranges overlap between nodes. Each [validator_start, validator_end) interval must be disjoint from every other node's interval." + } +} diff --git a/terraform/msf-2/outputs.tf b/terraform/msf-2/outputs.tf new file mode 100644 index 00000000..d252f291 --- /dev/null +++ b/terraform/msf-2/outputs.tf @@ -0,0 +1,120 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// GENERATED FILES AND OUTPUTS +//////////////////////////////////////////////////////////////////////////////////////// + +resource "local_file" "ansible_inventory" { + content = templatefile("ansible_inventory.tmpl", + { + ethereum_network_name = "${var.ethereum_network}" + groups = merge( + { for group in local.digitalocean_vm_groups : "${group.group_name}" => true... }, + { for group in local.hetzner_vm_groups : "${group.group_name}" => true... }, + ) + hosts = merge( + { + for key, server in digitalocean_droplet.main : "do.${key}" => { + ip = "${server.ipv4_address}" + ipv6 = try(server.ipv6_address, "none") + group = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^group_name:", tag))][0], "unknown") + validator_start = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^val_start:", tag))][0], 0) + validator_end = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^val_end:", tag))][0], 0) + supernode = try(title([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^supernode:", tag))][0]), "undefined") + arch = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^arch:", tag))][0], "amd64") + builder_index = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^builder_index:", tag))][0], "") + tags = "${server.tags}" + hostname = "${split(".", key)[0]}" + cloud = "digitalocean" + region = "${server.region}" + } + }, + { + for key, server in hcloud_server.main : "${key}" => { + ip = coalesce(server.ipv4_address, (try(server.ipv6_address, ""))) + ipv6 = coalesce(server.ipv6_address, "") + group = server.labels.group_name + validator_start = server.labels.val_start + validator_end = server.labels.val_end + supernode = server.labels.supernode + arch = server.labels.arch + builder_index = try(server.labels.builder_index, "") + tags = server.labels + hostname = split(".", key)[0] + cloud = "hetzner" + region = server.datacenter + } + } + ) + } + ) + filename = "../../ansible/inventories/${join("-", slice(split("-", var.ethereum_network), length(split("-", var.ethereum_network)) - 2, length(split("-", var.ethereum_network))))}/inventory.ini" +} + +locals { + ssh_config_path = pathexpand("~/.ssh/config.d/ssh_config.${var.ethereum_network}") +} + +resource "local_file" "ssh_config" { + content = templatefile("${path.module}/ssh_config.tmpl", + { + ethereum_network = var.ethereum_network + hosts = merge( + { + for key, server in digitalocean_droplet.main : "${var.ethereum_network}-${key}" => { + hostname = server.ipv4_address + private_ip = server.ipv4_address_private + name = key + user = "devops" + } + }, + { + for key, server in hcloud_server.main : "${var.ethereum_network}-${key}" => { + hostname = coalesce(server.ipv4_address, (try(server.ipv6_address, ""))) + private_ip = try(hcloud_server_network.main[key].ip, "") + name = key + user = "devops" + } + } + ) + } + ) + filename = local.ssh_config_path + + depends_on = [digitalocean_droplet.main, hcloud_server.main] + + lifecycle { + create_before_destroy = true + } +} + +resource "null_resource" "ssh_config_cleanup" { + triggers = { + ssh_config_path = local.ssh_config_path + } + + provisioner "local-exec" { + when = destroy + command = "rm -f ${self.triggers.ssh_config_path} || true" + } + + depends_on = [local_file.ssh_config] +} + +output "ssh_config_file" { + value = "SSH config generated at: ${local.ssh_config_path}" + description = "Path to the generated SSH config file" +} + +output "digitalocean_server_count" { + value = length(digitalocean_droplet.main) + description = "Number of DigitalOcean servers created" +} + +output "hetzner_server_count" { + value = length(hcloud_server.main) + description = "Number of Hetzner servers created" +} + +output "total_server_count" { + value = length(digitalocean_droplet.main) + length(hcloud_server.main) + description = "Total number of servers created across all providers" +} diff --git a/terraform/msf-2/ssh_config.tmpl b/terraform/msf-2/ssh_config.tmpl new file mode 100644 index 00000000..d2394b4d --- /dev/null +++ b/terraform/msf-2/ssh_config.tmpl @@ -0,0 +1,16 @@ +# SSH Config for ${ethereum_network} devnet +# Generated by Terraform +# Usage: Include this file in your ~/.ssh/config with: +# Include /path/to/this/ssh_config +# Or copy entries to your main SSH config file + +%{ for host_key, host in hosts ~} +Host ${host_key} + HostName ${host.hostname} + User ${host.user} + Port 22 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + LogLevel ERROR + +%{ endfor ~} From 1d607b519ce2e2edc3a7e1820920ee63bb436e9b Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 12 Oct 2026 10:21:56 +0200 Subject: [PATCH 2/2] msf-2: boot the cleaned release at Gloas genesis Restore the published snapshots at block 24452538, retain the release Amsterdam timestamp and start Gloas at 200M gas. Keep the per-client restore layouts and Nethermind TTD override, fix validator graffiti, and use the available SFO disk region. Apply the current ingress-secret split for platform ArgoCD access and clean up all pre-genesis gate temporary files. Genesis time remains unset for launch scheduling. --- .gitignore | 1 + .../msf-2/group_vars/all/all.sops.yaml | 16 +- .../inventories/msf-2/group_vars/all/all.yaml | 89 ++++----- .../msf-2/group_vars/all/ingress.sops.yaml | 170 ++++++++++++++++++ .../msf-2/group_vars/bootnode.yaml | 2 +- .../inventories/msf-2/group_vars/erigon.yaml | 4 +- .../inventories/msf-2/group_vars/ethrex.yaml | 2 +- .../inventories/msf-2/group_vars/geth.yaml | 4 +- .../msf-2/group_vars/grandine.yaml | 2 +- .../msf-2/group_vars/lighthouse.yaml | 2 +- .../msf-2/group_vars/lodestar.yaml | 2 +- .../msf-2/group_vars/nethermind.yaml | 3 + .../inventories/msf-2/group_vars/nimbus.yaml | 2 +- .../msf-2/group_vars/nimbusel.yaml | 2 +- .../inventories/msf-2/group_vars/prysm.yaml | 2 +- .../inventories/msf-2/group_vars/teku.yaml | 4 +- ansible/playbook.yaml | 1 + ansible/shadowfork.yaml | 3 +- ansible/templates/snapshot_download.sh.j2 | 7 +- scripts/normalize-shadowfork-genesis.py | 8 +- scripts/shadowfork-pregenesis-check.sh | 33 +++- terraform/msf-2/digitalocean.tf | 2 +- 22 files changed, 269 insertions(+), 92 deletions(-) create mode 100644 ansible/inventories/msf-2/group_vars/all/ingress.sops.yaml diff --git a/.gitignore b/.gitignore index 8e346d4e..aa2b8fd4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ scripts/*.json # derived genesis artifacts (too large for git) network-configs/devnet-9/parsed/** +__pycache__/ diff --git a/ansible/inventories/msf-2/group_vars/all/all.sops.yaml b/ansible/inventories/msf-2/group_vars/all/all.sops.yaml index 9d761b56..9c6b54af 100644 --- a/ansible/inventories/msf-2/group_vars/all/all.sops.yaml +++ b/ansible/inventories/msf-2/group_vars/all/all.sops.yaml @@ -1,13 +1,6 @@ secret_zerossl: ACME_EAB_KID: ENC[AES256_GCM,data:UAbtTHRnXHispR1iQr/5eQMV7JMk5g==,iv:WkDvniP4a1cjpRTQqT0MnJo9UkIp+iwJ3NPSTb8sFsU=,tag:yXavQCYBVJsPHwmm2WbwSg==,type:str] ACME_EAB_HMAC_KEY: ENC[AES256_GCM,data:V7U8TluhY0y550IjMfubUgHSFCdxqS109TwWjSGx818J2So5zrTJR0TWzRl1bH4/9MyqCpEtTLHzLlL0j7I4XzgNvrm8qjpeNw3yRa94eyWH5EJT8VI=,iv:Evd5W8ORLtUa3sSl6XeaqY69p0FAmpiFmjXDlP+XbUY=,tag:A34c1/jnfKSFjuDf/2stYA==,type:str] -secret_prometheus_remote_write: - username: ENC[AES256_GCM,data:sLeD3W2HobqS99rstG9qukc=,iv:H5QuY4h5RZxcoBg6qOR/uaWGeHsEDmQJ8uipsi1GF0Q=,tag:CiXtlhhH6/Jz/n2k2E+NFQ==,type:str] - password: ENC[AES256_GCM,data:FsxhGsp3XUSWEZbJVYFv2V6L5HYEv6mNSfxdRWPZSkOvTG4B,iv:JkK+me7CBgP44gwHwZdCfpd5ZKhUmG9Pr7uPelW0JuM=,tag:LjbmI9MHSeyLkPl1Vg5Nyw==,type:str] -secret_loki: - endpoint: ENC[AES256_GCM,data:fgNKy2JFJ+eWraqSN8mmrMD2qua/od9Ai4KkWt0BaqJuJeNbkisVERcjZXL/OFZK0eJqNVzQh+w=,iv:3hdFH0Y+33EXpCMoEhQHMj1cwxn2EMFOdPj2YwJZAhg=,tag:H6KbRsCGIC0axq4qFfP++A==,type:str] - username: ENC[AES256_GCM,data:hX2WNfHo3UumyB9y7ti+9JY=,iv:zRvSJabBqxANzCOixupm9ghIMriKQTrqlzkLAFy22fk=,tag:umcsnxdjasOYZ6achgw0kw==,type:str] - password: ENC[AES256_GCM,data:Lup8xKhAXpbSjGU1aICYM4CR0jHzm0FYSz4a38H/psHLSHln,iv:576Quu09HRoruCzO9l1j6b9wlO0er/NKe1HTxNJudzc=,tag:+Vb16lFEFYR4JfpvOYZV1w==,type:str] secret_nginx_shared_basic_auth: name: ENC[AES256_GCM,data:O7L3,iv:DiOGlqfOfrDlt7X4OGY27OYlkDDEHguv+kg1zRhBek0=,tag:KuOX20lI/iYZOORFraiISw==,type:str] password: ENC[AES256_GCM,data:HyMDeoK5s6JDfsJ7j5Gg,iv:yvC4WWOOJvVeegWl0uj2P8yxcfBgNrvYSm5xUB5qux8=,tag:XoZbXpvLtUJPVoEaV3ltsg==,type:str] @@ -44,13 +37,8 @@ secret_dockerhub: token: ENC[AES256_GCM,data:eSROwx/YdGm+LxygbpG1Zrqftilp5ogfXkVpratOBQId0D3a,iv:/1UyFsGqodzel2opDTR62ZVbRvQMB6EGCmehv9SlxZ8=,tag:0S/W4eXpZlVJ893u1eTpvg==,type:str] secret_buildoor_wallet_privkey_2: ENC[AES256_GCM,data:PhlAnCwX6x/8IT7LNJMHET5lcJ5CfeHjbz6eesvqTf0A4xVo5+7DLEAoukjVDtHF2uPSlVVHaYU7ASUs+Z/IzQ==,iv:97Z90S9tUfzmGRW+QI5mvJ/Ou+13BRB2+MGGNp/EILQ=,tag:G9B84vf4iU31bvCrZlUcCQ==,type:str] sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2026-10-07T19:35:45Z" - mac: ENC[AES256_GCM,data:6RyuG02q61vn169JQAX+MVMxqaTJZe2qTj66f3V4l1cpGrtZrl9mzZXYpRiQIOpudMXfthqEVKpc8pqPYkrBRG9Upd08H2VPmq5oEdXwkpNIMPDCpHiiHsaUo+/QbHS643st6UFnDlm6681PLIrmFbmR5xE/CYJBOLmPfTe4Bjg=,iv:m1fp/HdETtBQ3hWc5YAlMpvzZiYmKvuRkBAHMH8J58Q=,tag:TFh42jOr9yUiLE9HObHCDQ==,type:str] + lastmodified: "2026-10-12T08:33:26Z" + mac: ENC[AES256_GCM,data:fJ8yzi42hZQ9PiKs5Q4MSl1Y+7GuCQ9YcGbRD6yFYz5+lnq2ABd1//yoiFMEEBK6pZIyC0NORURWt0dNbDmC3uAsDa/l3W7Mwu+aSg3odSijkMUOELTTzVsAqQz/dfs9v1csHblB9e5HL9uCkO2cgd9Es53tLkqfZGa9wySSVic=,iv:cEfVqfnRGSQQofqdux4Ctu2PPvv2DmYMvgatLBbPSTU=,tag:UZMIv1zqgRA6RR2+xEtkpw==,type:str] pgp: - created_at: "2025-10-27T13:25:35Z" enc: |- diff --git a/ansible/inventories/msf-2/group_vars/all/all.yaml b/ansible/inventories/msf-2/group_vars/all/all.yaml index abb44b7a..d75aace4 100644 --- a/ansible/inventories/msf-2/group_vars/all/all.yaml +++ b/ansible/inventories/msf-2/group_vars/all/all.yaml @@ -53,19 +53,29 @@ ethereum_genesis_network_seed: glamsterdam-msf-2 ethereum_genesis_chain_id: "1" ethereum_genesis_fork_version_suffix: "{{ 999999 | random(start=100000, seed=ethereum_genesis_network_seed) }}" ethereum_genesis_generator_output_dir: ../network-configs/{{ network_iteration }} -# 2026-10-07 19:00 UTC -ethereum_genesis_timestamp: 1791399600 +# TODO(msf-2): ~45 min after every EL is restored +ethereum_genesis_timestamp: 0 ethereum_genesis_timedelay: 0 -# jochemnet (benchmarkoor): mainnet 24350000 + 1 s bloat blocks up to 24402727, not on -# mainnet. Gate before genesis: eth_config shows our amsterdamTime on every EL. -# shadowfork_height is the snapshot; the head is 2 blocks later, built on a restored geth and -# imported into every EL over the engine API (scripts/shadowfork-prestate/): -# 24402728: benchmarkoor's funding (15001 withdrawals of 2^64-1 gwei, funding.txt of -# gas-benchmarks amsterdam-repricings-v5.2.0) + 1 ETH for a throwaway deployer -# 24402729: the two EIP-8282 contracts via the 0x4e59 CREATE2 factory -shadowfork_height: 24402727 -shadowfork_head_height: 24402729 -gloas_fork_epoch: 40 +# The jochemnet release (benchmarkoor): mainnet 24350000 + bloat blocks to 24402727, then the +# release pre-run to 24443820 (crosses Amsterdam at 1769856769), ramped down to 200M, cleaned and +# re-exported per client to shadowfork-images.ethpandaops.io/jochemnet///. Not on +# mainnet, so no mainnet txs replay. +# The release head: 24443820 + the 8718-block ramp to 200M (built 2026-10-08/11, head hash +# 0x0fd7242bab5055dda36f374f352c78cae381daae9b3e8febe895dce35c907591). +shadowfork_height: 24452538 +# No shadowfork_head_height: nothing is imported on top of the snapshot (msf-1 had a prestate). +# The snapshot is already on Amsterdam, so the CL starts on Gloas (no live transition, builders +# from genesis) and genesis.json carries the release's own amsterdamTime: egg would put it at +# genesis, after the release's Amsterdam blocks, and every EL would rewind them. +shadowfork_snapshot_root: "{{ shadowfork_snapshot_base_url }}/{{ snapshot_fetcher_network }}/geth/{{ shadowfork_height }}" +# 1769856769 for the jochemnet release. +shadowfork_amsterdam_time: >- + {{ (lookup('ansible.builtin.url', shadowfork_snapshot_root ~ '/_snapshot_metadata.json', split_lines=false) + | from_json).shadowfork.amsterdam_time }} +gloas_fork_epoch: 0 +# 36 supernodes x 1000 keys (terraform/msf-2/nodes.tf). A reduced fleet passes its own total +# with -e, or finality never comes (participation is counted against this number). +ethereum_genesis_validator_count: 36000 ethereum_genesis_mnemonic: "{{ secret_genesis_mnemonic }}" ethereum_genesis_generator_config_files: cl/config.yaml: "{{ lookup('ansible.builtin.url', 'https://raw.githubusercontent.com/ethpandaops/ethereum-genesis-generator/v{{ethereum_genesis_generator_version}}/config-example/cl/config.yaml', @@ -77,21 +87,21 @@ ethereum_genesis_generator_config_files: # Network identity and fork schedule only - every spec parameter not listed # here follows the egg defaults, which match sepolia's Gloas config.yaml. # Shadowfork specifics: - # - SHADOW_FORK_FILE: eth_getBlockByNumber("0x1745b29", true) after the prestate import (full - # txs; jochemnet publishes none, the one in the erigon tarball is block 24350000). + # - SHADOW_FORK_FILE: the release head's _snapshot_eth_getBlockByNumber.json (full txs), + # published next to every snapshot. # - DEPOSIT_CONTRACT_ADDRESS must be mainnet's or the generator refuses. - # - GAS_LIMIT_SCHEDULE: 200M at the Gloas epoch, like sepolia. + # - GAS_LIMIT_SCHEDULE: 200M from genesis (the Gloas epoch). # - WITHDRAWAL_TYPE=0x01 + VALIDATOR_BALANCE=10000 ETH (every fusaka shadowfork # did this): shadowfork mode adds no premines, so the 9968 ETH excess per # validator is swept to WITHDRAWAL_ADDRESS (mnemonic index 0, # 0x454b0EA7d8aD3C56D0CF2e44Ed97b2Feab4D7AF2) and funds the tooling from there. # Index 0 is empty on real mainnet. # Not set because shadowfork mode ignores them: EL premines, GENESIS_GASLIMIT, - # DEPLOY_EIP8282_CONTRACTS (deployed in block 24402729, see shadowfork_head_height). + # DEPLOY_EIP8282_CONTRACTS (deployed by the release pre-run). values.env: |- export CHAIN_ID="{{ ethereum_genesis_chain_id }}" export EL_AND_CL_MNEMONIC="{{ ethereum_genesis_mnemonic }}" - export NUMBER_OF_VALIDATORS=36000 + export NUMBER_OF_VALIDATORS={{ ethereum_genesis_validator_count }} export GENESIS_FORK_VERSION="0x10{{ ethereum_genesis_fork_version_suffix }}" export ALTAIR_FORK_VERSION="0x20{{ ethereum_genesis_fork_version_suffix }}" export BELLATRIX_FORK_VERSION="0x30{{ ethereum_genesis_fork_version_suffix }}" @@ -108,13 +118,13 @@ ethereum_genesis_generator_config_files: export VALIDATOR_BALANCE=10000000000000 export DEPOSIT_CONTRACT_ADDRESS=0x00000000219ab540356cBB839Cbe05303d7705Fa export SHADOW_FORK_FILE=/config/shadowfork_block.json - # Block 24402729 from the bootnode geth (eth_getBlockByNumber, full txs). - shadowfork_block.json: "{{ lookup('ansible.builtin.file', (ansible_inventory_sources[0] | dirname) ~ '/files/shadowfork_block.json') }}" + shadowfork_block.json: "{{ lookup('ansible.builtin.url', shadowfork_snapshot_root ~ '/_snapshot_eth_getBlockByNumber.json', split_lines=false) }}" # noqa yaml[line-length] # role: ethpandaops.general.snapshot_fetcher (variable names reused by # ansible/shadowfork.yaml, which downloads with parallel ranged GETs instead). # snapshot_fetcher_out_dir/_filename are set per EL in its group_vars file. snapshot_fetcher_network: jochemnet +shadowfork_snapshot_base_url: https://shadowfork-images.ethpandaops.io snapshot_fetcher_client: "{{ ethereum_node_el }}" snapshot_fetcher_block: "{{ shadowfork_height }}" @@ -258,13 +268,9 @@ gen_kubernetes_config_disabled_services: - faucet # Its "add to MetaMask" offers chain id 1: a tx signed there with a real key is valid on mainnet. - homepage -# The EIP-8282 contracts are in block 24402729, so this only checks them (it skips the deploy -# when the deposit contract has code). -gen_kubernetes_config_assertoor_tests: - - id: deploy-eip8282-contracts - file: https://raw.githubusercontent.com/ethpandaops/assertoor/master/playbooks/gloas-dev/deploy-eip8282-contracts.yaml - schedule: - startup: true +# No deploy-eip8282-contracts startup test: it refuses after Gloas, which is genesis here. The +# release snapshot must hold both contracts (scripts/shadowfork-pregenesis-check.sh checks the code). +gen_kubernetes_config_assertoor_tests: [] # role: ethpandaops.general.xatu_sentry xatu_sentry_container_image: "{{ default_tooling_images.xatu_sentry }}" @@ -567,32 +573,5 @@ vector_config: | # Static EL peers (erigon, ethrex, nimbus-el): mainnet peers have none of our blocks. # Firewall EL p2p on every host right after -t init-server (before any EL starts): # FW_HOSTS='ethereum_node:bootnode' scripts/shadowfork-ethrex-firewall.sh msf-2 -# The 27 geth/besu/reth/nethermind ELs, from the first playbook run's ethereum_node_fact_discovery_el_enode facts (2026-10-07). -shadowfork_el_static_enodes: - - enode://182aa43c0045e4dc6a59c55e08bf15b08d72646b8c96cf6de63b63e2ece6074eea440eb65340552a71db93c501028f4996dea8695ba02d52877236fde34e2220@165.22.214.79:30303 # bootnode-1 - - enode://4d4cd5e3a249e0e68277da4512307cd948fd56465546f466e92dcf03d9d1dc7dec837cc4882e36a61a63eedf6db0f0f9ab9bc1e784121a7a0643b04b0024be11@134.122.118.11:30303 # buildoor-lighthouse-geth-1 - - enode://0b66e95f8c543a4b4e4e16351ffdf9370b91ac36908e20c6cfe5d2457dbdc5314efada923bdb0f276885e1efb06ebbcbcf836fa9198b3ba07d1809086293a453@165.232.143.60:30303 # buildoor-teku-nethermind-1 - - enode://543215cc00c4705e26e32c684cd441984755611ff8d782d20595d2498d9d5d3bc51486f513fdf811f8bf47a294a3b72f21f4263f044a78366ff158598386b25e@162.243.160.238:30303 # grandine-besu-1 - - enode://861731a9793525326d38fae0fd2406b61383f025a7f035ff309de431785c3d0b7bdbc0e325c5e807ce0c6cad3dd5bda94608dce2a0f5f1dc5796a2267710f3ab@139.59.142.22:30303 # grandine-geth-1 - - enode://5d4dd99be4f0450a6dca4d5d5600a6ab8d3b6df3ff099b183a94b7117e3de25c4c04293dad52f9d088b94c5c85511ff56584069f00baed3b1e09a9b00821d159@137.184.152.222:30303 # grandine-nethermind-1 - - enode://1555805631467b69f511a1e6ab34701c3a367644878cbb8375fa22ae5640405f64c22885a5c2a629ce9b3ea87f92369d365a355a6969b769492799a8e2087ccb@165.232.180.28:30303 # grandine-reth-1 - - enode://2929d39acab48742bbffe21691f605aea7fcd6f47e7a2eb381857f35a6ddcc3f81ba9c17e150eda54d34eab8f1aad9a04d0c53f12b426bb8b1f7ebd3feb02dcc@146.190.143.108:30303 # lighthouse-besu-1 - - enode://1f7c633d559b09361b418885270946def404976d18cf31bfe6ec2a80d01c544c95697384e6da2879d98c4076ef882c3fa7ef365e88ee288c9e2a6637e8eb4ea3@134.209.177.39:30303 # lighthouse-geth-1 - - enode://d10489d056d158ad3387e025f6b61f220991dd79fe16a54c5362267b2e4975da78e723f4b9b6d44c3d74c5bcd2287c34318a95fb6baf02aa6226f07793eaf621@138.197.134.58:30303 # lighthouse-nethermind-1 - - enode://b4f270b1c07ee9568d0762741dd3b058fa22983da1a94195c99cc72ce5c1c0f5aaead0739fe08146801696d3913b69e85c91914b8149d297ed012c575f3335bb@165.22.42.44:30303 # lighthouse-reth-1 - - enode://62ce527ef2ec32df22ac6d884f0990bce7df26d602df610156dbe04fc0cb7a29b0063630e29185a65379924a42f9308490dbc69c25b9cd78e262a50a981b223f@201.79.6.41:30303 # lodestar-besu-1 - - enode://dd70c7191fdeae64c38f4b2bf6797575d2a9db9f244917af3e8f292a72ab323cdb06be7a46ebeffb7fb01d13fb8d31be7bce4214b8d8719ee5329b1bd617c3d8@188.166.82.109:30303 # lodestar-geth-1 - - enode://989de4d1d045e6e948246207ae98592a05f4c6e03f2b2a6a66f0ab891488205cdaafe6635e1e78cf3702b2252a9f8c60c04f049f8b11e73b3efa7eaa41e2cd1c@143.198.193.222:30303 # lodestar-nethermind-1 - - enode://767876e92d10f00f8a056ed3f77f8c46630005bd886066afd23659b710dde1d6d5f133469a8325a7849cfad860caa60a23e38f510ec1d291ac3493f94dfaf69c@159.65.186.218:30303 # lodestar-reth-1 - - enode://630ca24e819f30fb7f2fc748a942b03b0e242b4c9f485c685cceccbbdc84720d0589064270874de8e328a91f9a05b3b83264d1bbce0d04eb9275230cd6169ce9@159.223.78.111:30303 # nimbus-besu-1 - - enode://f42a3a4d5428c62dcc77bae868a630e0e167a6c93f13bb9ec77f7740edbcd0636896a6cdb112401fd6d95776d0eed9ac68a5a13c33fcbeaf0d911f2bd555fbbb@168.144.149.58:30303 # nimbus-geth-1 - - enode://c6338a35aa889787464156f80b4f966afeb383c12b8b0e96fe69d193d3fbfbdd31c73aba109624b9d26202c0401391e21fe0c37dc207bfa517450f6644165ff1@147.182.149.145:30303 # nimbus-nethermind-1 - - enode://c2bc8f2f5a84a43de9a4fd118f2fbf85e800ffbd2f09ba70d2cb1e3c655a961fb9e5c8c35a30bcfc6e098d7e0b7ba42c74cafd340b1284d9c78bd330929f33c5@159.203.86.74:30303 # nimbus-reth-1 - - enode://1a28067a6124ebd021bd36e112e5f7c3d29813972be7c8b40a9ae1a957c83c56e69502130620e701a68fff8d7b370ca012da7ef058f3738324d1a5b094bc1555@178.62.223.33:30303 # prysm-besu-1 - - enode://3b8a724028715338fd5813bbc0ca252391f93ae7899f9f5544234d667662e77cf662a85feda767bfd72e4e10764022767c67971417200cb8f12a646e746dc5db@139.59.12.219:30303 # prysm-geth-1 - - enode://c85042645cdf1e4e068d6ff0bce83555302c3e6b0d084d2bfafc82dbda43e64f29201a08642870cfcc91a03bd1cae70c07aa7c64e143a39d92433a3ad465a2e9@159.65.115.85:30303 # prysm-nethermind-1 - - enode://1daa8dd39ef039d4522c0ca3b83115bdad6062610485358b3d731610d6bd3b0d6fc82eb758445caa7ec6d2b16eb38a4ad7270f7fbc05dcdb88f9cc97be7709e2@143.110.218.78:30303 # prysm-reth-1 - - enode://ba7d05b484905ae4ccebcc96adde5111fcf5d88f2490f453190b2ea123dbb08370beb5639a30ee3dc12382134545efa2db3738d67dc884e4bd84d29a3a426594@138.197.121.104:30303 # teku-besu-1 - - enode://1095ea3a6f63612d1f32d403de348bb51c5867780485256b49cca69d6623495db8a87f0795b2fb69e217a2153077e468db1d2c1d3ca75e1fcfb9568b67a238e2@24.199.107.252:30303 # teku-geth-1 - - enode://9ac088d35ebd5908c665d9ff567e411747bad6124bfaa7a081e80f345be23e3cf94acfac44e110b2723ae728986b312bdf0b2083c1d46280c04c6d9cfc50d8aa@46.101.101.144:30303 # teku-nethermind-1 - - enode://6b9487acea9aa05c38adaa0fa61a0866564fc96797a754cf8b3d27fbbcad9d516484729812789b14d538303fe26377ec7c5c4d57c2bdba988d8c58caf665d6d7@178.128.203.221:30303 # teku-reth-1 +# TODO(msf-2): the ELs' ethereum_node_fact_discovery_el_enode facts after the first playbook run. +shadowfork_el_static_enodes: [] diff --git a/ansible/inventories/msf-2/group_vars/all/ingress.sops.yaml b/ansible/inventories/msf-2/group_vars/all/ingress.sops.yaml new file mode 100644 index 00000000..ee2c782b --- /dev/null +++ b/ansible/inventories/msf-2/group_vars/all/ingress.sops.yaml @@ -0,0 +1,170 @@ +secret_prometheus_remote_write: + username: ENC[AES256_GCM,data:YbOHqvv1L3fvo1lCqv6Qafk=,iv:eDmp0uGciteNmZ4nRJtHDKja3pPrL3LItzhhzQLy4ks=,tag:902PklO7w29wuQLSSwnmpg==,type:str] + password: ENC[AES256_GCM,data:04+mgtY7mlS/XgztX7BQeR3fq+T1FP93294b+jciwO2LhiIB,iv:errdcuIR9jn3SSKNZfX29x/eXj8KYvf1sYtEyXrq7J8=,tag:HwXnQkI7+AtKL4Mmx4axBg==,type:str] +secret_loki: + endpoint: ENC[AES256_GCM,data:N7SwGm1IxXXKmf8CM9CAUcOng6VzT2wnkYLu1oVRimt70qAKgT1Oe8U7FrcHp7okCDI9SoJz83I=,iv:zrO92WEk0Iwqyh1vLI6Jnn5f5RjsYv00osss4jN/UG8=,tag:lNF96VLt3F5/6XjvcCk5bg==,type:str] + username: ENC[AES256_GCM,data:ohK5N/1bzWAHH6FqJTKMX2A=,iv:NTvsNzAE/mGHnfwTApe7r3XRfMVrSZq6s5vLNGJzvP8=,tag:sGKrAjYUuvu9WQfvMBNuUA==,type:str] + password: ENC[AES256_GCM,data:xkwJPqhvcav7zynX1DMCHJP6rgydniBVgPDrhgC6XHgS4Pbu,iv:ngE7MjdC2Mqf8gYvYCXPi87bU2ymUACLCgjnvrkzC6c=,tag:tu9fD+h1O0lLeOIF2N5cgw==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2NU56VWRhaXVzem9DenRK + WUdaSXBlQlpKVUQ3NW5STlppTmExaDQwVzFzClM3ZEJYU0NramM1Ykx3UDZlVDJ0 + OXBaSzZ6Sm11SENRbVNCME9HYzdIeEkKLS0tIFVpVy9tbnR6V3ZkeGVDSStXTjhP + cUlIVTdXNjdDc1ZlbjB5elFOTEtYdGsKeMoBY6cJvnVpwq+KpxBQC3WRYjZoW5K7 + O1NCwGwyQvekikpHOcV1bD8G6WDyC22+o58lryyBgbZwmq0e9hjVaQ== + -----END AGE ENCRYPTED FILE----- + recipient: age1jzmeqmpu70anvkt78pl5y0w67r6jpdkamr4cr95mts9vthjly58qfctt55 + lastmodified: "2026-10-12T08:33:25Z" + mac: ENC[AES256_GCM,data:ELyweGyeBkfJ2blQamkI5ijSKyOFeTSGr35cm11FVB31W1bVS23gsYjUSVOnfoMQhDkHHaAfxC+LAcTtiNTJjjx0DFVEs2liydvG/30DlF0kw1NxP0TJxh5hlMXMNjqbVD5EpTz9xGfJxDemLflcN+cTgtiJi+KuYC3Gz8n/A04=,iv:JcexWIIhPMwQ5gkXnuCCDy3Ym+CULhusWdBxUOzJKLA=,tag:vHKrwRs891BMYHeLfzmSOw==,type:str] + pgp: + - created_at: "2026-10-08T05:14:44Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMA+2+1l9DpxI8ARAAgwD0Nr1P1k9DfOcrTlEVCz6SKHZVg9Qew8CUXjN6Qhj8 + /wZkkhpdzL03nPx7G5TONpwz8SoGZqmj26wZps2lV3Mg5zbcXUJaoUjH84EaSNKP + beSErJk3DIsEVj2S6RjDi6DVUhUad6Sgi+h3r9DHmkV5vuQoSmHsaW/4EGboIafp + IUHQbFkbB/Yo18c1nqKjUN9gGs3CA1E2aXFVT10kH/tszYCYH7sdkrbXNPFxk5Cz + MFOaCEQ235yylLU4zMSYudFV4YIhjoCKNDAKKZdX3uyVFyWJ38HuIAABK1rurGPs + ywgEqUAk00YVeOXjhceDroE/zg7akm/GcCkHzLGtZa7DUi1wqbIjgFe/qF8vLNcE + HNxW3JhPap26sQd4bMfZXG4zpXZDQRqkchpn2Jb2eddJ8IllGzNAHdu/HMwwi9vS + 5pjM0Et0613b6aBMYRAZL2fbMAlhZvHPz/qZNq32rO1LwLTb/e4omZezyJJxy1HU + jxmYYRgd5inh2FlLaUSq3O+vbIiLhWUpwWIpovzhji3+YiSW+4N8wx3HG0+NHkJr + LLZw/0B26YUTGxkTgxthy8l9IFYuZpyn76mzgn+omIIxUs6zguHIZktDtkmynIWB + ECfcflrgHtlF0mOQdyP5XlIaojrbz5Duocgsb/f09PeUYu8Ub23xIV3n7Q3+lU/S + UQHqpVHelbKFAnlrdspC/O3kHsRJ4FvjMb211pOv7+qKejJ9HdKz4yocRUM+dn2b + v3BsiH2TqTPcjbniUIapwcOYLPxL1SyPZfhCDefAhiEfig== + =ewwU + -----END PGP MESSAGE----- + fp: 80DB2B4EF6CA4D9829C280605636BC0E08138A24 + - created_at: "2026-10-08T05:14:44Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMA87Wber4r4dGARAA2VNQG7jDrt11gdtF0wLXhboPjG6fr15UhlPhMnUtmDPU + q9h8dWEC05yhUKKXkpnUGUa1totutkLNfvBFYZovRP08XN575cXPaesEeQn4LW65 + s9pcORGnWYq+yv4LhTqP8rmXd31RoPlroLGOAxAMi0dtIvVPcdzgcOsf+9xO4A1J + AUSaKKOJcSpiiW6R/kogCvH9fLu224qE4lmM76VokkvGDLgmuvowQrhg0wXnMW7/ + m6n0bzekn4INvt0JdHO7KlDwRoUmfwTB6uLFuWV6EwIgseCPNvGWWeWf5KB/mheL + LyjKae3xOvS4LhpucLqi6hzXeenfceta1DFYu2181t5zz0LhwYhgff0/d7Pw/tt3 + A0arOu6YbY9jndZAJOjGOW1+II9dcOB/Y2CI+mdoA5uz/hDhIxCBGTGDJG9L/dg7 + 2ck9a4xx1PIvIbhv15/XEXQslsHptccgZ8f2VdKqcg8sHneRD1tMAoDojR3Ca6xB + xa69cSXkce49q7jj4roqpaLdrree8ZvwGo2a599pVGOu/ggvtMlpYSSqyA9MMxRy + 7xadC/oKw3EirwM4qkSKWOWi3eJLcE8DTDTN2Iv1zMvlwZzihjTdaEHFZdWLWy3V + xbPCLQDkOGPx8dCy/WZcBI33PFMEa4OihLKFOVsqHUHjq/3alJl9jZdONjSPgBXS + UQG+I3GWauGtugjpVQiisVuPZQI/iYMx0WHggBzfTmpJCbRZ02cJk3BOyOWQzfjE + kev/dTjpBEiYbNg8hZlglVlvEDPetYB3pQwuSL+VUwNimw== + =MFQY + -----END PGP MESSAGE----- + fp: 69F66EEA7AE36CCB77DDB8CA1BC39532FB4A2DBD + - created_at: "2026-10-08T05:14:44Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQIMA4hw3nPn82LyAQ//YkOq7isYFlxk2HSep4HL7++2GXMIz0twqH5GInnlIupE + oR3gaarHFc6LZp4pWusZzypA1qmx1m3f2dAIEXaul88KIgDygZlpZ0gbVnH1okHJ + u97FcLrZyErF2YiyPqdoKk9NIdcBX586/1QKYSBW+0Dsjgyvc+XrRIyaBD1bqlUj + j43DCWKhnHj4yX7cHIFg6n6ylnjkCMB1X5dHDB0l4KMiRRv7KEvv19UM4oNE4NiE + DoBEkJZSbrO7yhuWxPhyWYwG9/5BgzMLSwCWtsM+bEGfnjXQ/VboKAPvXuYi1rxs + gJMQR7hzBmiZj3TsWvG4DVcDEzrfSp6za0KYhcHGULyF6A3yKkJxPLYF77uD6iGI + z7GWpkyvGvMAzJzrokYiMDDkuYBdKRuo477rT813KzJKuViNKpihiuoirCEpYg24 + 1UtG5Klraax14aQgQWqShU5yfk1CjKj5aXC7h9r9UTnAQ0tOaosLmpMJ2oLq3Y66 + tNBysc+CoAfz2y/n2v2bti/3nax+EUTdNuPpLja65I3rFTY4OR/nzeXNwzQFdjgt + L6Z9DosNuloGJ9/nJHr+xXx9M9xkn7FDiw3i4186NwrBa2dZR2FkX56Z/ur/Row/ + pXJIzaWYLBX9ubtoKeIoPSjHCfEKQ0u0Onrn3kE2rus7fmnhgaPWyHcBduqLgevS + UQGRjXyBGFc5HlJ0NH34SHqID5pN0oM/1Fgn3fvcFVqycv6tR7oWYPPVbX1wAGo4 + f+E1pX0h6bmqD29wjhX3bESqCzClznYdFo9TofngqsbvrQ== + =5bkj + -----END PGP MESSAGE----- + fp: D1002590180DE371BCB455EAEFCD4ACD0B4D3F6E + - created_at: "2026-10-08T05:14:44Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMAw01qfIBwY0JARAAo+8j8sVE/od7A1KiPtEraccdOGSZcZk4nvGINWPKOctf + VR6X4RRf34iXIpobqkFr6DcFTDcAgTP7FOhsfX9uZbFtDjcGx5JRiqYLu+7c3sh8 + YVyJ+Id1rdwFdzqoi49eDWNXO8NOnGojd/uuw+nfYEg7aFgfXcjKWfoonSm1GbxI + dO95OfpYI7dH95zzYS84sdbWCjuGH772cgbc6f3BarI9yatxA9LI8Oz3LI+5a/hH + KaY58L5s1Hr9Y/C6L201+ZZmYDBKwZjiKU/XCcS+c5IiD6ChNOC3C3RRhVVJA2Iq + kMBozAooP10NtZ78xWf571n6L/z/VVBFLvySiNufsS7SVSxaDOWC4cXSkeJ81N/r + HGIsObF6d1g8ByP/JMzYOHlWdwcgwjRYn1aQtGp9rk+2+bv6JVbKEMCzzYAgooRg + EzyiGXYHhUnjc00WvBuqPgXyufLZPf3PKl0lAUutrs//LeRAFp0zqYdY9EIfHS3X + I5rqQP7t2OIQ9t7m8mBdgzhmIDPgK85ZUoyIRIdn/7vKjDuwmvHRxLaQFjRo30WQ + ApTxFY5/3DUhMeMmGbRy/+VbWYk+hGq6hQoLvIdohPNWKR+Dh65kuz4H46sQsOwM + ZcZqgmBop4UOrptaHMlt+5szt1hotY6WrDx2LNZiPizIUmY8n9twEnUYW67A297S + UQFGXtY67ELR+dXYtlrMV4E4CzUsCj7wTcEJMpza5XDOKHaTPWtJlyFLmvtSev7Y + goNdzXK72CeiIq0BleTT7ZWonErj5+TV0iDb9qFZgsSJGg== + =LeA7 + -----END PGP MESSAGE----- + fp: B9F81F327CF5346860E85269D7AF98F214C59E4E + - created_at: "2026-10-08T05:14:44Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcBMAwDFOgk9tOQ+AQf8DitTS7os8kkHDMk+IHBYtHmNDOjTDo4LB2L09bIqdEZG + Ml1M59gPjm9nE2gs1YDb9S7kLGvHiXKWeAVRox8TWv2PMSw7FyXenrAIbo+Jz3Fm + 1GzL32Dr0SfXWghCKccIzHzjt+atpBT+/waHc8Wcn5KT2ns0iqGPfry/Ib1WQHBa + J0HdK5COz/TRczLm0JvNUCqCp+Azn18uE4Xmq7NsHooFufmAQD22mp5BceSkS6qA + q158tyZR5woTqi8Fh2+2vfT6+NLXpPCcCgPOEzHuFy7Ia5o3M2Ox3wcFZmL64Aj5 + pDr8yDHnvqSserONeGQrCpe44VgRMV26m9cC2CmvptJRAZiPAl3Zj70HVZiTLbuj + ZY7po8JX/Z+8ACNrN+QvUknq8Pzj5BNmNqb/+2dAxdHPnBc258gMayWnq0HFTe2a + MONomB8o3JUqD+Jn2Tg9dtrO + =HOlz + -----END PGP MESSAGE----- + fp: 0600D41E1313E31016F7C55BF597BC0C5F22D1A2 + - created_at: "2026-10-08T05:14:44Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMAz4a8AV36xppAQ/+Ny/Eu7U3/1sV9gwMAP4Y6sc8wSvjHdCJ1zikGhUsss3Q + P8AQ6w0uv/mlX6uW95aNRXK3p/Gi8kluPWgToU8vRj6UgGow3zDHKD09qL4naRhY + /RgohORHwq2fPh0zqjB65WrRkFjRdEOllaiXFMZySXJyrVIeL89qWSlFZpAEIy7K + 0OgeoqyVUjQ4+jtJUnjeGeQtPMIUkPATybp3NqgCNRzo1xaF7CoVmdcA+H06IgDu + P0Gg4Xc1SZDCoIsFakFOiLpcI7cw5iB38lt+xRuYmxjg8OixmmakwHtOno2G47M+ + Og9uZUKX4GcrgJJX4kqwhmCQjwqpMTGuwEhX5jx+rJJLiK7PQNtnRu2ZEcDRZIx3 + dm0K0JTS69xb73p9xsEtiOqBkaFxlrnRw651x508FyHB+GHB4cBBRqaiopltRF+k + P1AyZbhAHH665LmN1SJguQAHdhxOL9ivXuMzx/bY/8Tlc1EiLyN6kc0bDVuCI22a + LTqcdBC5fHqw3owoMB6tC/933aYt7auYt3xtrj/bXgTxrevG4+j8lI3fdMQU1jDN + AaKwPFIgegWZmoEKCLB5mQwlvkWrEnM9zD8iHt9SK5WyuHVlwJ7/2AIcOnG7M7ou + fT1x+0zJc4hGLvyXUfjhPgHdOBDapx/f1EaqmlxVd7Nwum91g94TXQzkhNwN2gTS + UQEDl8Bb/mGrH3xc6darlbuRjdFb5eRMjjP3hvH49bJLDeKKRVQcLCuQG06QbEcM + 3kjBNexVazuJqOTqlzeUhioiS8HcdoY5E04I9X4T03BpSw== + =X1Zf + -----END PGP MESSAGE----- + fp: 29C50D01122FDE78E257482DAA497EB2610A8435 + - created_at: "2026-10-08T05:14:44Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQIMAxYzhHRfYJYtAQ//RH/HrRU2Z8bt6BisTWvAJZEyeAN7wX+1umyPvq+VjXzd + qUjF+4VlOSQAolaUsqNLN3GoeEywPl3zchHwNGkPgL38x8+uUbaMH0cHEH2rwzSo + rvJHHZk7gNtQ7N8kxAn58FztWIsRoaz4AFa+bDawnLRkZKqgykCfEZgahZCb0KvA + V/j0gpycLUF/bvk7JGW6VLLzJtb7jgyoK73fdLQiIhBOT7MTLlLMh8zaGeEY+w5E + 928toiPC+tgG9GDXfm1Whu2DdOhaSxePyBJ279Hs0CcZZhqb8grg/iE+dQOlVYdx + z/SQ5fFBLPoF6+8WSn8wH7toUwrSyc6rlysni1OveA6qRO5lSuREbMtItyPsEa5g + f3kQ0XQ3RmxqAzBJGIuyueI+BLrBfZ+TvPdLMfSG6b7nO94DBFxCZiGz+fBEw+xP + rFClK6cMSr6Hqemw44Fc/oIQCKFfyD/CQpZg4LAKIGyifcy3WuKi6RPYRRQI+227 + mXshl1bZOslloECelhbGss9MjElI3OEILnuUH709pLM0TpFg/K1/wC7f+U8sexog + 0ojLqFqAbcPOaWmwxxHxc7Fc8ZAOFZohjB6ozFMAh+BvMJaxPrTlUzsXzzOwUo4/ + jN0xR+S63T5JxYxBaOwio8mLY7sBSBNOwwJHeX2YhZl7zM7Ri2vRPfegKwy8n2XS + XgGUA4omZwChGwv79H+oisZeYeSRjKo+tCKNRYd82anPJz5DdYDHz69d/Jgg2EXf + gBs5x580874/HL++50Jd3hKeDR+PyALAnKT6OyugoITjZyzlUv8DlrzzJhGjqE0= + =AMXH + -----END PGP MESSAGE----- + fp: 9BE537027CB7467923E240FF2AED09371C121F91 + - created_at: "2026-10-08T05:14:44Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hF4DjS7VXZx9i8YSAQdAwa3GLBJnXhzEyy1eW4ubpIzeg8uVW1iCbd6omquimF8w + QEFoi8X7I/+hsOE0Olcve3DWOrpK7yrKOsTnNzzRx1cL/eU5FAH8zR/fhTN+4lxY + 1GgBCQIQ3EdpyggQtaby9Tj4kT5j/jiLIE+MzJejB5PjrZRDOPos75feXY2efWdC + /ge4GqtuRQoxPhzK3AWHlJ0snxUg6o1emJ1rPmNGEygvVMPPmqYHSEKp0CD300Mk + 9W+TDSYPYeJ1Cw== + =dd1Q + -----END PGP MESSAGE----- + fp: F93098C3D8ED15D0924A6DDAEB4E93A88660C55B + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/ansible/inventories/msf-2/group_vars/bootnode.yaml b/ansible/inventories/msf-2/group_vars/bootnode.yaml index 5680a8de..37907809 100644 --- a/ansible/inventories/msf-2/group_vars/bootnode.yaml +++ b/ansible/inventories/msf-2/group_vars/bootnode.yaml @@ -101,7 +101,7 @@ prysm_container_command_extra_bootnode_args: >- # role: ethpandaops.general.ethereum_node ethereum_node_el: geth -# role: ethpandaops.general.snapshot_fetcher (bootnode is not in the geth group) +# role: ethpandaops.general.snapshot_fetcher (bootnode is not in the geth group; see geth.yaml) snapshot_fetcher_out_dir: /data/geth/geth # role: ethpandaops.general.geth diff --git a/ansible/inventories/msf-2/group_vars/erigon.yaml b/ansible/inventories/msf-2/group_vars/erigon.yaml index 3577d7c0..189a7b42 100644 --- a/ansible/inventories/msf-2/group_vars/erigon.yaml +++ b/ansible/inventories/msf-2/group_vars/erigon.yaml @@ -7,8 +7,8 @@ bootstrap_default_user_authorized_keys_github_team_el: >- # role: ethpandaops.general.snapshot_fetcher snapshot_fetcher_out_dir: /data/erigon -# Frozen up to 24401000; retire starts ~300 blocks after genesis. If it stalls then (v3.7.1 -# retire deadlock), add --snap.stop via erigon_host_extra_args. +# Frozen up to 24401000; the ~43k release blocks above that are in the DB, so retire starts soon +# after genesis. If it stalls (v3.7.1 retire deadlock), add --snap.stop via erigon_host_extra_args. snapshot_fetcher_filename: snapshot-pruned.tar.zst # role: ethpandaops.general.ethereum_node diff --git a/ansible/inventories/msf-2/group_vars/ethrex.yaml b/ansible/inventories/msf-2/group_vars/ethrex.yaml index 8b1c1caf..c883399c 100644 --- a/ansible/inventories/msf-2/group_vars/ethrex.yaml +++ b/ansible/inventories/msf-2/group_vars/ethrex.yaml @@ -9,7 +9,7 @@ ethereum_node_fact_discovery_el_key_cmd: ethrex: xxd -p -c32 {{ ethrex_datadir | default('/data/ethrex') }}/chain-{{ ethereum_network_id | trim }}/node.key # role: ethpandaops.general.snapshot_fetcher (bare RocksDB; ethrex 29 reads /chain-, -# hardcoded: genesis.json does not exist at restore time). The May db is migrated on start. +# hardcoded: genesis.json does not exist at restore time). snapshot_fetcher_out_dir: /data/ethrex/chain-1 # role: ethpandaops.general.ethereum_node diff --git a/ansible/inventories/msf-2/group_vars/geth.yaml b/ansible/inventories/msf-2/group_vars/geth.yaml index fb2f09d0..0ed56e1b 100644 --- a/ansible/inventories/msf-2/group_vars/geth.yaml +++ b/ansible/inventories/msf-2/group_vars/geth.yaml @@ -5,7 +5,9 @@ bootstrap_default_user_authorized_keys_github_team_el: >- {{ (query('ethpandaops.general.url_cached', coredevs_api_url ~ '/api/v1/users/geth?format=txt') | select | list) + geth_extra_github_users }} -# role: ethpandaops.general.snapshot_fetcher (snapshot layout: chaindata/ at the top) +# role: ethpandaops.general.snapshot_fetcher: a geth archive is geth's instance dir, unprefixed +# (snapshots.ethpandaops.io and benchmarkoor snapshot export alike), so it goes under /geth. +# Extracted flat, geth opens chaindata/ but misses triedb/merkle.journal and rewinds its head. snapshot_fetcher_out_dir: /data/geth/geth # role: ethpandaops.general.ethereum_node diff --git a/ansible/inventories/msf-2/group_vars/grandine.yaml b/ansible/inventories/msf-2/group_vars/grandine.yaml index 4dfcb4cf..f25feab7 100644 --- a/ansible/inventories/msf-2/group_vars/grandine.yaml +++ b/ansible/inventories/msf-2/group_vars/grandine.yaml @@ -46,7 +46,7 @@ grandine_container_command_extra_args: >- grandine_container_command_extra_simple_args: - --configuration-directory=/network-config/ - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }} - - --graffiti={{ ansible_hostname }} + - --graffiti={{ inventory_hostname }} - --features=LogHttpRequests grandine_validator_container_volumes: diff --git a/ansible/inventories/msf-2/group_vars/lighthouse.yaml b/ansible/inventories/msf-2/group_vars/lighthouse.yaml index 430ff109..89b41192 100644 --- a/ansible/inventories/msf-2/group_vars/lighthouse.yaml +++ b/ansible/inventories/msf-2/group_vars/lighthouse.yaml @@ -47,7 +47,7 @@ lighthouse_validator_container_volumes: - "{{ eth_testnet_config_dir }}:/network-config:ro" lighthouse_validator_container_command_extra_args: - --testnet-dir=/network-config - - --graffiti={{ ansible_hostname }} + - --graffiti={{ inventory_hostname }} - --builder-proposals lighthouse_validator_datadir: /data/lighthouse-validator lighthouse_container_pull: true diff --git a/ansible/inventories/msf-2/group_vars/lodestar.yaml b/ansible/inventories/msf-2/group_vars/lodestar.yaml index d041953f..c18911c3 100644 --- a/ansible/inventories/msf-2/group_vars/lodestar.yaml +++ b/ansible/inventories/msf-2/group_vars/lodestar.yaml @@ -50,6 +50,6 @@ lodestar_validator_container_volumes: - "{{ eth_testnet_config_dir }}:/network-config:ro" lodestar_validator_container_command_extra_args: - --paramsFile=/network-config/config.yaml - - --graffiti={{ ansible_hostname }} + - --graffiti={{ inventory_hostname }} lodestar_validator_datadir: /data/lodestar-validator lodestar_container_pull: true diff --git a/ansible/inventories/msf-2/group_vars/nethermind.yaml b/ansible/inventories/msf-2/group_vars/nethermind.yaml index 914197f2..851de24a 100644 --- a/ansible/inventories/msf-2/group_vars/nethermind.yaml +++ b/ansible/inventories/msf-2/group_vars/nethermind.yaml @@ -32,6 +32,9 @@ nethermind_container_command_extra_args: - --Init.ChainSpecPath=/network-config/genesis.json - --Init.BaseDbPath=nethermind_db/mainnet - --FlatDb.Enabled=true + # The replay uses TTD 0; stored synthetic headers can have TD below mainnet TTD. + # Keep them post-merge during startup re-execution as well as Engine API import. + - --Merge.TerminalTotalDifficulty=0 - --Sync.FastSync=false - --Sync.SnapSync=false - --JsonRpc.EnabledModules=Eth,Subscribe,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Admin diff --git a/ansible/inventories/msf-2/group_vars/nimbus.yaml b/ansible/inventories/msf-2/group_vars/nimbus.yaml index 11b4c7a8..c0714f45 100644 --- a/ansible/inventories/msf-2/group_vars/nimbus.yaml +++ b/ansible/inventories/msf-2/group_vars/nimbus.yaml @@ -50,7 +50,7 @@ nimbus_container_command_extra_simple_args: - --doppelganger-detection=off - --log-level=DEBUG - --light-client-data-serve=true - - --graffiti={{ ansible_hostname }} + - --graffiti={{ inventory_hostname }} # Gloas: the CL sets the payload gas target (PayloadAttributesV4.targetGasLimit) and # nimbus v26.9.1 ignores GAS_LIMIT_SCHEDULE (default 60M), so its blocks pulled the gas # limit back toward 60M after the fork while every other CL ramps toward 200M. diff --git a/ansible/inventories/msf-2/group_vars/nimbusel.yaml b/ansible/inventories/msf-2/group_vars/nimbusel.yaml index fa9d4d96..07041c8c 100644 --- a/ansible/inventories/msf-2/group_vars/nimbusel.yaml +++ b/ansible/inventories/msf-2/group_vars/nimbusel.yaml @@ -21,7 +21,7 @@ nimbusel_container_volumes: - "{{ eth_testnet_config_dir }}:/network-config:ro" # Start these only after Gloas has finalized (playbook.yaml -l '!nimbusel' before): v0.4.2 # snap sync needs a post-Amsterdam pivot (bals_download.nim:78) and the static enodes. -# No jochemnet nimbus-el snapshot: snap-sync from the restored ELs (experimental, hidden +# No nimbus-el snapshot: snap-sync from the restored ELs (experimental, hidden # --debug-snap-sync in v0.4.2). Peer only with our snapshot ELs: real mainnet peers share # our fork ID until amsterdamTime and would serve mainnet's state. Discovery off, # one --static-peers per enode; p2p is also firewalled (scripts/shadowfork-ethrex-firewall.sh). diff --git a/ansible/inventories/msf-2/group_vars/prysm.yaml b/ansible/inventories/msf-2/group_vars/prysm.yaml index 2f399d24..fc2219f2 100644 --- a/ansible/inventories/msf-2/group_vars/prysm.yaml +++ b/ansible/inventories/msf-2/group_vars/prysm.yaml @@ -60,6 +60,6 @@ prysm_validator_container_entrypoint: prysm_validator_container_command_extra_args: - --chain-config-file=/network-config/config.yaml # Hosts that set validator_display_name stamp it; the rest keep the hostname. - - --graffiti={{ validator_display_name | default(ansible_hostname) }} + - --graffiti={{ validator_display_name | default(inventory_hostname) }} - --enable-builder prysm_validator_datadir: /data/prysm-validator diff --git a/ansible/inventories/msf-2/group_vars/teku.yaml b/ansible/inventories/msf-2/group_vars/teku.yaml index 748c4a63..6a47194e 100644 --- a/ansible/inventories/msf-2/group_vars/teku.yaml +++ b/ansible/inventories/msf-2/group_vars/teku.yaml @@ -46,7 +46,9 @@ teku_container_command_extra_simple_args: - --genesis-state=/network-config/genesis.ssz - --ignore-weak-subjectivity-period-enabled - --p2p-discovery-bootnodes={{ ethereum_cl_bootnodes | join(',') }} - - --validators-graffiti={{ ansible_hostname }} + # inventory_hostname, not ansible_hostname: the machine is named - (35 bytes for + # teku-nethermind-1) and teku rejects graffiti over 32 bytes, so the container restart-loops. + - --validators-graffiti={{ inventory_hostname }} - --logging=info - --Xlog-include-p2p-warnings-enabled - --metrics-block-timing-tracking-enabled diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index 04af61bd..3dedf083 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -26,6 +26,7 @@ ansible.builtin.command: >- python3 {{ playbook_dir }}/../scripts/normalize-shadowfork-genesis.py {{ ethereum_genesis_generator_output_dir }}/metadata/genesis.json + {{ shadowfork_amsterdam_time | default('') }} args: chdir: "{{ playbook_dir }}" register: normalize_genesis diff --git a/ansible/shadowfork.yaml b/ansible/shadowfork.yaml index 11dd27c0..ec03b978 100644 --- a/ansible/shadowfork.yaml +++ b/ansible/shadowfork.yaml @@ -11,7 +11,8 @@ - hosts: ethereum_node:bootnode:!localhost become: true vars: - snapshot_fetcher_base_url: https://snapshots.ethpandaops.io + # play vars beat group_vars, so an inventory picks the host with shadowfork_snapshot_base_url + snapshot_fetcher_base_url: "{{ shadowfork_snapshot_base_url | default('https://snapshots.ethpandaops.io') }}" snapshot_fetcher_container_image: docker.ethquokkaops.io/dh/library/alpine:3.19.1 snapshot_fetcher_workers: 16 snapshot_fetcher_chunk_bytes: "{{ 64 * 1024 * 1024 }}" diff --git a/ansible/templates/snapshot_download.sh.j2 b/ansible/templates/snapshot_download.sh.j2 index c444e7e1..56f83f05 100644 --- a/ansible/templates/snapshot_download.sh.j2 +++ b/ansible/templates/snapshot_download.sh.j2 @@ -61,10 +61,11 @@ wait $! [ ! -f "$T/failed" ] || { echo "download failed, datadir is incomplete" >&2; exit 1; } # Snapshots come from nodes that ran: drop their p2p identity and saved peers so every # host gets its own node id and nothing dials the parent network. geth/erigon nodekey + -# nodes/, reth discovery-secret + known-peers.json, besu key, ethrex node.key + -# node_config.json, nethermind /peers + discoveryNodes. +# nodes/ (geth/ too: an archive packed with the geth/ prefix), reth discovery-secret + +# known-peers.json, besu key, ethrex node.key + node_config.json, nethermind +# /peers + discoveryNodes. cd /data -for p in nodekey nodes discovery-secret known-peers.json key node.key node_config.json */peers */discoveryNodes; do +for p in nodekey nodes geth/nodekey geth/nodes discovery-secret known-peers.json key node.key node_config.json */peers */discoveryNodes; do [ ! -e "$p" ] || { rm -rf "$p"; echo "removed $p"; } done touch /fetcher/done diff --git a/scripts/normalize-shadowfork-genesis.py b/scripts/normalize-shadowfork-genesis.py index 5e466df9..e4838e09 100755 --- a/scripts/normalize-shadowfork-genesis.py +++ b/scripts/normalize-shadowfork-genesis.py @@ -7,7 +7,11 @@ Address not found"). Verified on geth/reth/erigon/nethermind/besu/ethrex/nimbus-el, 2026-10-07. Prints "changed" when it rewrote the file. -Usage: normalize-shadowfork-genesis.py +With an amsterdamTime argument it also sets config.amsterdamTime: a datadir restored past +Amsterdam (msf-2) needs the fork where its blocks put it, not at genesis, or every client +rewinds them as pre-Amsterdam blocks. + +Usage: normalize-shadowfork-genesis.py [amsterdamTime] """ import json import sys @@ -23,6 +27,8 @@ config = genesis["config"] config.setdefault("withdrawalRequestContractAddress", "0x00000961ef480eb55e80d19ad83579a64c007002") config.setdefault("consolidationRequestContractAddress", "0x0000bbddc7ce488642fb579f8b00f3a590007251") +if len(sys.argv) > 2: + config["amsterdamTime"] = int(sys.argv[2]) if json.dumps(genesis, sort_keys=True) != before: with open(path, "w") as f: diff --git a/scripts/shadowfork-pregenesis-check.sh b/scripts/shadowfork-pregenesis-check.sh index ae24b820..5d163d19 100755 --- a/scripts/shadowfork-pregenesis-check.sh +++ b/scripts/shadowfork-pregenesis-check.sh @@ -2,8 +2,8 @@ # Pre-genesis gate for a shadowfork: run after playbook.yaml started the ELs and before the # CL genesis time. Exits non-zero if any check fails; every line says which host and why. # genesis.json: alloc keys 0x, request contract addresses, only the expected fork keys -# per EL host (except nimbusel): head == shadowfork_head_height (else shadowfork_height) with the shadowfork_block.json hash, eth_config -# next activation == amsterdamTime, unique node id, peers only fleet IPs, +# per EL host (except nimbusel): head == shadowfork_head_height (else shadowfork_height) with the SHADOW_FORK_FILE hash, eth_config +# next (or current) activation == amsterdamTime, unique node id, peers only fleet IPs, # /data < 85%, erigon snapshots/preverified.toml, ethrex chain-1/metadata.json # # Usage: scripts/shadowfork-pregenesis-check.sh @@ -16,13 +16,27 @@ inv="$root/ansible/inventories/$net/inventory.ini" vars="$root/ansible/inventories/$net/group_vars/all" block_json="$root/ansible/inventories/$net/files/shadowfork_block.json" meta="$root/network-configs/$net/metadata" +temp_block_json="" +ids="" +cleanup() { + [ -z "$temp_block_json" ] || rm -f "$temp_block_json" + [ -z "$ids" ] || rm -f "$ids" +} +trap cleanup EXIT +height="$(yq '.shadowfork_head_height // .shadowfork_height' "$vars/all.yaml")" +if [ ! -f "$block_json" ]; then + # msf-2 reads SHADOW_FORK_FILE from the snapshot bucket, not the inventory + url="$(yq '.shadowfork_snapshot_base_url + "/" + .snapshot_fetcher_network' "$vars/all.yaml")/geth/$height/_snapshot_eth_getBlockByNumber.json" + temp_block_json="$(mktemp)" + block_json="$temp_block_json" + curl -fsS "$url" -o "$block_json" || { echo "error: $url not found" >&2; exit 2; } +fi for f in "$inv" "$block_json" "$meta/genesis.json"; do [ -f "$f" ] || { echo "error: $f not found" >&2; exit 2; }; done fail=0 bad() { echo "FAIL $*"; fail=1; } ok() { echo "ok $*"; } -height="$(yq '.shadowfork_head_height // .shadowfork_height' "$vars/all.yaml")" want_hash="$(jq -r '.result.hash' "$block_json")" amsterdam="$(jq -r '.config.amsterdamTime' "$meta/genesis.json")" printf 'expect head %s %s, amsterdamTime %s\n' "$height" "$want_hash" "$amsterdam" @@ -53,17 +67,26 @@ fleet="$(grep -oE 'ansible_host=[0-9.]+' "$inv" | cut -d= -f2 | sort -u)" hosts="$(cd "$root/ansible" && ansible -i "$inv" 'ethereum_node:bootnode:!nimbusel' --list-hosts 2>/dev/null | tail -n +2 | tr -d ' ')" rpc() { curl -sf --max-time 15 -X POST -H 'content-type: application/json' \ --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$2\",\"params\":$3}" "${fmt//\{host\}/$1}"; } -ids="$(mktemp)"; trap 'rm -f "$ids"' EXIT +ids="$(mktemp)" for h in $hosts; do b="$(rpc "$h" eth_getBlockByNumber '["latest",false]' || true)" num="$(jq -r '.result.number // empty' <<<"$b" 2>/dev/null)"; hash="$(jq -r '.result.hash // empty' <<<"$b" 2>/dev/null)" if [ -z "$num" ]; then bad "$h: no RPC answer"; continue; fi [ "$(( num ))" = "$height" ] && [ "$hash" = "$want_hash" ] && ok "$h head $height" \ || bad "$h head $(( num )) $hash" - next="$(rpc "$h" eth_config '[]' | jq -r '.result.next.activationTime // empty' 2>/dev/null || true)" + # a post-Amsterdam snapshot (msf-2) has no next fork: amsterdam is current + next="$(rpc "$h" eth_config '[]' | jq -r '(.result.next // .result.current).activationTime // empty' 2>/dev/null || true)" if [ -z "$next" ]; then echo "warn $h: no eth_config, check amsterdamTime in its startup log" elif [ "$(( next ))" = "$amsterdam" ]; then ok "$h eth_config next $amsterdam" else bad "$h eth_config next activation $(( next )) != $amsterdam"; fi + # EIP-8282: every Amsterdam block is invalid without code at both addresses. A pre-Amsterdam + # head may still get them from the assertoor deploy; a post-Amsterdam one must already hold them. + for a in 0x0000bff46984e3725691fa540a8c7589300d8282 0x000064d678505ad48f8ccb093bc65613800e8282; do + code="$(rpc "$h" eth_getCode "[\"$a\",\"latest\"]" | jq -r '.result // empty' 2>/dev/null || true)" + if [ "${#code}" -gt 2 ]; then ok "$h code at $a" + elif [ "$(( $(jq -r '.result.timestamp' <<<"$b") ))" -ge "$amsterdam" ]; then bad "$h: no code at $a past Amsterdam" + else echo "warn $h: no code at $a yet (deploy before Gloas)"; fi + done rpc "$h" admin_nodeInfo '[]' | jq -r --arg h "$h" '.result.id // empty | "\(.) \($h)"' >> "$ids" 2>/dev/null || true # every public IPv4 in admin_peers must be ours (private/container addresses are skipped) foreign="$(rpc "$h" admin_peers '[]' 2>/dev/null | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}' | sort -u \ diff --git a/terraform/msf-2/digitalocean.tf b/terraform/msf-2/digitalocean.tf index 0eaf4ff6..8db13db5 100644 --- a/terraform/msf-2/digitalocean.tf +++ b/terraform/msf-2/digitalocean.tf @@ -31,7 +31,7 @@ variable "digitalocean_regions" { "fra1", "tor1", "blr1", - "sfo3" # syd1 dropped: no so1_5-8vcpu-64gb-intel there + "sfo2" # so1_5-8vcpu-64gb-intel is unavailable in sfo3 (2026-10-12) ] }