From b7055eaddc0ed0db088bbb8c26a82e9bf1adb2c8 Mon Sep 17 00:00:00 2001 From: alexhulbert Date: Wed, 27 May 2026 21:31:43 -0400 Subject: [PATCH 01/17] Bump gcp measure script version --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index c1f4c0cf..468c4efd 100644 --- a/flake.nix +++ b/flake.nix @@ -44,8 +44,8 @@ src = pkgs.fetchFromGitHub { owner = "flashbots"; repo = "dstack-mr-gcp"; - rev = "ecf3284b72a507fd005de91d49f7372490cf6995"; - sha256 = "sha256-1rcm9sIZuvCojNN2HMPrsECYn9sd8eVChsgwleo8nFY="; + rev = "ed23e96785ebfb1ff153503b01cfbfb10cffae67"; + sha256 = "sha256-jrHcAEp4OYmOMTJ1BWIULoqKycqqQBWIRLjhKmiZor4="; }; vendorHash = "sha256-glOyRTrIF/zP78XGV+v58a1Bec6C3Fvc5c8G3PglzPM="; }; From 70f2d20b5792cc7fbbb7662e8e7cf1b56a8a75fe Mon Sep 17 00:00:00 2001 From: Pablo <118397961+pablin-10@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:41:45 -0300 Subject: [PATCH 02/17] Prometheus metrics setup (#125) --- images/flashbox-l1.conf | 1 + .../mkosi.extra/usr/bin/init-container.sh | 6 ++ .../mkosi.extra/usr/bin/init-firewall.sh | 1 + .../common/mkosi.extra/usr/bin/toggle | 11 +++- .../mkosi.extra/etc/bob/firewall-config | 6 ++ modules/flashbox/observability/mkosi.conf | 8 +++ .../etc/prometheus/process-exporter.yml | 5 ++ .../etc/prometheus/prometheus.yml.tmpl | 45 ++++++++++++++ .../etc/prometheus/recording_rules.yml | 45 ++++++++++++++ .../flashbox-observability-setup.service | 14 +++++ .../etc/systemd/system/node-exporter.service | 26 ++++++++ .../systemd/system/process-exporter.service | 19 ++++++ .../etc/systemd/system/prometheus.service | 29 +++++++++ .../usr/bin/flashbox-observability-setup | 59 ++++++++++++++++++ .../mkosi.extra/usr/lib/flashbox/vault.sh | 62 +++++++++++++++++++ modules/flashbox/observability/mkosi.postinst | 12 ++++ 16 files changed, 346 insertions(+), 3 deletions(-) create mode 100644 modules/flashbox/observability/mkosi.conf create mode 100644 modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml create mode 100644 modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl create mode 100644 modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml create mode 100644 modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service create mode 100644 modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service create mode 100644 modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service create mode 100644 modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service create mode 100755 modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup create mode 100644 modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh create mode 100755 modules/flashbox/observability/mkosi.postinst diff --git a/images/flashbox-l1.conf b/images/flashbox-l1.conf index 42f61ce8..a5d07dfd 100644 --- a/images/flashbox-l1.conf +++ b/images/flashbox-l1.conf @@ -2,6 +2,7 @@ Include=shared/mkosi.conf Include=modules/flashbox/common/mkosi.conf Include=modules/flashbox/flashbox-l1/mkosi.conf +Include=modules/flashbox/observability/mkosi.conf [Config] Profiles=azure,gcp diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh b/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh index b4251905..221c84b8 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh +++ b/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh @@ -8,6 +8,9 @@ SEARCHER_SSH_PORT=10022 SEARCHER_INPUT_UDP_PORT=27017 # External UDP input channel SEARCHER_INPUT_TCP_PORT=27018 # External TCP input channel +# ENDPOINTS +PROMETHEUS_PROXY_IP=10.88.0.100 # host firewall allows in ALWAYS_OUT + # Run extra commands which are customized per image, # see bob*/mkosi.extra/etc/bob/searcher-container-before-init # @@ -73,6 +76,9 @@ ns_iptables() { ns_iptables -A OUTPUT -d 169.254.169.254 -j DROP +# Block container from reaching the internal Prometheus Proxy +ns_iptables -A OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP + # Block consensus layer P2P port (TCP and UDP) ns_iptables -A OUTPUT -p tcp --dport 9000 -j DROP ns_iptables -A OUTPUT -p udp --dport 9000 -j DROP diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh b/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh index 8701b55e..2e7f3a01 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh +++ b/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh @@ -151,6 +151,7 @@ drop_dst_ip() { # # `source` is not supported in dash ########################################################################### + . /etc/bob/firewall-config ########################################################################### diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/toggle b/modules/flashbox/common/mkosi.extra/usr/bin/toggle index 088001a7..13f5f0df 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/toggle +++ b/modules/flashbox/common/mkosi.extra/usr/bin/toggle @@ -45,6 +45,9 @@ CHAIN_MAINTENANCE_OUT="MAINTENANCE_OUT" CHAIN_PRODUCTION_IN="PRODUCTION_IN" CHAIN_PRODUCTION_OUT="PRODUCTION_OUT" +# Endpoints +PROMETHEUS_PROXY_IP="10.88.0.100" + # Source image-specific configuration if available TOGGLE_CONFIG="/etc/bob/toggle-config" if [ -f "$TOGGLE_CONFIG" ]; then @@ -217,11 +220,13 @@ check_searcher_namespace_rules() { && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27017 -j DROP 2>/dev/null \ && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27018 -j DROP 2>/dev/null \ && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p udp --dport 123 -j DROP 2>/dev/null \ - && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null; then - echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018) are ACTIVE." + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null \ + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d 169.254.169.254 -j DROP 2>/dev/null \ + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP 2>/dev/null; then + echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are ACTIVE." return 0 else - echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123) are NOT found." + echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are NOT found." return 1 fi } diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index 4d88d139..84e95993 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -32,6 +32,9 @@ FLASHBOTS_BUNDLE_2="3.15.88.156" FLASHBOTS_TX_STREAM_1="3.136.107.142" FLASHBOTS_TX_STREAM_2="3.149.14.12" +# Prometheus metrics proxy +PROMETHEUS_PROXY_IP="10.88.0.100" + ########################################################################### # (1) ALWAYS_IN: Inbound rules that are always applied ########################################################################### @@ -74,6 +77,9 @@ accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $TITAN_IP $TITAN_BUNDLE_PORT "Titan bui # they can observe the presence or absence of packets. accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $FLASHBOTS_BUNDLE_1,$FLASHBOTS_BUNDLE_2 $HTTPS_PORT "Flashbots bundle" +# Prometheus metrics proxy (searcher netns has a DROP on this IP — see init-container.sh) +accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $PROMETHEUS_PROXY_IP $HTTPS_PORT "Prometheus metrics proxy" + ########################################################################### # (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode ########################################################################### diff --git a/modules/flashbox/observability/mkosi.conf b/modules/flashbox/observability/mkosi.conf new file mode 100644 index 00000000..f45d7c01 --- /dev/null +++ b/modules/flashbox/observability/mkosi.conf @@ -0,0 +1,8 @@ +[Content] +ExtraTrees=modules/flashbox/observability/mkosi.extra +PostInstallationScripts=modules/flashbox/observability/mkosi.postinst + +Packages=prometheus + prometheus-node-exporter + prometheus-process-exporter + gettext-base diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml b/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml new file mode 100644 index 00000000..033f901d --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml @@ -0,0 +1,5 @@ +process_names: + # Monitor the searcher container (conmon + all children via --children flag) + - name: "searcher-container" + cmdline: + - 'conmon.*searcher-container' diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl b/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl new file mode 100644 index 00000000..c570fec0 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl @@ -0,0 +1,45 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + # Stamped onto every series sent via remote_write so the upstream can + # distinguish samples coming from different flashbox VMs. + external_labels: + host: ${FLASHBOX_VM} + +# Recording rules for aggregated metrics +rule_files: + - /etc/prometheus/recording_rules.yml + +scrape_configs: + # Node exporter on localhost + - job_name: 'node' + static_configs: + - targets: ['localhost:9100'] + metric_relabel_configs: + # Only keep aggregated metrics for remote write + - source_labels: [__name__] + regex: 'node_(cpu|memory|disk|filesystem|network|vmstat)_.*' + action: keep + + # Process exporter for container monitoring + - job_name: 'process' + static_configs: + - targets: ['localhost:9256'] + +# Remote write via the internal Prometheus metrics proxy at the fixed proxy IP, +# so no DNS resolution happens on the VM; TLS SNI + cert validation are +# overridden via tls_config.server_name to match the cert the proxy's upstream +# presents. +remote_write: + - url: https://${PROMETHEUS_PROXY_IP}/workspaces/${METRICS_FLASHBOTS_WORKSPACE}/api/v1/remote_write + write_relabel_configs: + # Only send flashbox: prefixed metrics + - source_labels: [__name__] + regex: 'flashbox:.*' + action: keep + tls_config: + server_name: aps-workspaces.${METRICS_FLASHBOTS_REGION}.amazonaws.com + sigv4: + region: ${METRICS_FLASHBOTS_REGION} + access_key: ${METRICS_FLASHBOTS_ACCESS_KEY} + secret_key: ${METRICS_FLASHBOTS_SECRET_KEY} diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml b/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml new file mode 100644 index 00000000..91475c7c --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml @@ -0,0 +1,45 @@ +groups: + # Base metrics — local: prefix means they stay inside the TEE + # (remote_write only forwards flashbox:*) + - name: local_container_metrics + interval: 30s + rules: + # Percent of total machine CPU capacity (normalized by core count, so the + # 0-100 scale holds regardless of how many vCPUs the VM has). + - record: local:container_cpu_percent + expr: > + sum(rate(namedprocess_namegroup_cpu_seconds_total{groupname="searcher-container"}[5m])) * 100 + / count(node_cpu_seconds_total{mode="idle"}) + + # Forwarded metrics — flashbox: prefix, picked up by remote_write + - name: flashbox_health + interval: 30s + rules: + # 1 if at least one process is running in the searcher-container group, + # absent if process-exporter isn't reporting it. + - record: flashbox:container_alive + expr: namedprocess_namegroup_num_procs{groupname="searcher-container"} > bool 0 + + # Spike-guarded: current 15m avg must be under 80%, + # AND the 10m max ending 5m ago must have been under 70% + - record: flashbox:container_average_cpu_is_under_80_percent + expr: > + (avg_over_time(local:container_cpu_percent[15m]) < bool 80) + * (max_over_time(local:container_cpu_percent[10m] offset 5m) < bool 70) + + - record: flashbox:container_oom_kills_count + expr: node_vmstat_oom_kill + + - record: flashbox:disk_free_space_is_over_10_percent + expr: > + (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) > bool 0.1 + + - record: flashbox:disk_free_space_is_over_128_gb + expr: > + (node_filesystem_avail_bytes{mountpoint="/persistent"}) > bool (128 * 1024 * 1024 * 1024) + + - record: flashbox:network_is_up + expr: > + (sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m])) + + sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))) + > bool 0 diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service new file mode 100644 index 00000000..a8fcf4f7 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service @@ -0,0 +1,14 @@ +[Unit] +Description=Flashbox observability setup (fetch creds, render Prometheus config) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/flashbox-observability-setup +RemainAfterExit=yes +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service new file mode 100644 index 00000000..d21a36bc --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service @@ -0,0 +1,26 @@ +[Unit] +Description=Prometheus Node Exporter +Documentation=https://github.com/prometheus/node_exporter +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus-node-exporter \ + --web.listen-address=127.0.0.1:9100 \ + --collector.disable-defaults \ + --collector.cpu \ + --collector.meminfo \ + --collector.diskstats \ + --collector.filesystem \ + --collector.netdev \ + --collector.loadavg \ + --collector.vmstat \ + --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker)($|/) +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service new file mode 100644 index 00000000..30b1257c --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service @@ -0,0 +1,19 @@ +[Unit] +Description=Prometheus Process Exporter +Documentation=https://github.com/ncabatoff/process-exporter +After=network-online.target searcher-container.service +Wants=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus-process-exporter \ + --web.listen-address=127.0.0.1:9256 \ + --config.path=/etc/prometheus/process-exporter.yml \ + --children +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service new file mode 100644 index 00000000..17eafe39 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service @@ -0,0 +1,29 @@ +[Unit] +Description=Prometheus Monitoring System +Documentation=https://prometheus.io/docs/introduction/overview/ +After=network-online.target flashbox-observability-setup.service +Wants=network-online.target + +# flashbox-observability-setup writes this file only when Vault creds are +# fetched + validated. If it's absent (Vault down, dev image, bad secret) the +# unit is cleanly skipped — inactive, not failed, no Restart= crash-loop — and +# self-heals on the next boot that renders a config. +ConditionPathExists=/etc/prometheus/prometheus.yml + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/var/lib/prometheus/ \ + --storage.tsdb.retention.time=24h \ + --web.console.templates=/usr/share/prometheus/consoles \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.listen-address=127.0.0.1:9090 +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup b/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup new file mode 100755 index 00000000..7cf4f37e --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup @@ -0,0 +1,59 @@ +#!/bin/bash +set -eu -o pipefail + +# Boot-time observability setup: authenticate to Vault, fetch + validate the +# Prometheus remote_write secret, and render /etc/prometheus/prometheus.yml. +# +# Prometheus ships flashbox:* metrics to Prometheus via the internal TCP-proxy +# at a fixed IP (10.88.0.100); the host firewall statically allows that IP, and +# TLS is validated via tls_config.server_name in the template. +# +# Fail-safe: on any failure (QEMU dev, Vault unreachable, missing/invalid +# secret) this writes NO config and exits 0. Because prometheus.service has +# ConditionPathExists=/etc/prometheus/prometheus.yml, it is cleanly skipped +# (inactive, not failed) until a later boot renders a config — boot never +# fails because of observability. + +PROMETHEUS_CONFIG=/etc/prometheus/prometheus.yml +PROMETHEUS_TEMPLATE=/etc/prometheus/prometheus.yml.tmpl + +# Internal metrics proxy IP, injected into the template so the literal isn't +# duplicated there. Must match the host firewall rule (firewall-config) and the +# container netns DROP (init-container.sh). +PROMETHEUS_PROXY_IP=10.88.0.100 +export PROMETHEUS_PROXY_IP + +. /usr/lib/flashbox/vault.sh + +# Always start clean — never leave a stale config from a previous boot. +rm -f "$PROMETHEUS_CONFIG" + +# host= external label, from the GCE instance name. +FLASHBOX_VM=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/name") || FLASHBOX_VM=unknown +echo "$FLASHBOX_VM" | grep -qE '^[a-z0-9-]+$' || FLASHBOX_VM=unknown +export FLASHBOX_VM + +# Vault auth uses the GCE instance-identity JWT, so it only works on GCP. +if ! grep -q Google /sys/class/dmi/id/sys_vendor 2>/dev/null; then + echo "Not running on GCP, leaving Prometheus disabled" + exit 0 +fi + +# Fetch + validate the four metrics variables (exported by vault_fetch). +if ! vault_fetch; then + echo "WARNING: Vault fetch/validation failed, leaving Prometheus disabled" + exit 0 +fi + +# The rendered config holds the SigV4 secret +touch "$PROMETHEUS_CONFIG" +chown root:prometheus "$PROMETHEUS_CONFIG" +chmod 640 "$PROMETHEUS_CONFIG" + +# Single render: only the curated variables are substituted; every other token in +# the template (e.g. [__name__]) is left untouched. +envsubst '$FLASHBOX_VM $PROMETHEUS_PROXY_IP $METRICS_FLASHBOTS_WORKSPACE $METRICS_FLASHBOTS_REGION $METRICS_FLASHBOTS_ACCESS_KEY $METRICS_FLASHBOTS_SECRET_KEY' \ + < "$PROMETHEUS_TEMPLATE" > "$PROMETHEUS_CONFIG" + +echo "Observability config written (workspace: ${METRICS_FLASHBOTS_WORKSPACE})" diff --git a/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh b/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh new file mode 100644 index 00000000..6bca21b6 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Vault GCP auth + fetch of the Prometheus remote_write secret. +# Sourced by flashbox-observability-setup. +# +# vault_fetch logs in with the GCE instance-identity JWT, reads the shared +# secret, and exports the four METRICS_FLASHBOTS_* vars. Each value is +# format-checked before export; any failure returns non-zero and the caller +# writes no config. + +vault_fetch() { + local addr mount role kv suffix jwt token data + + # curl --retry: this runs early in boot, so the metadata server and Vault + # may not be reachable on the first try. Without retries a transient miss + # would disable metrics until the next reboot. + + # 1. Bootstrap config from GCE instance metadata. + addr=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_addr") || return 1 + mount=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_auth_mount_gcp") || return 1 + role=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_role") || return 1 + kv=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_kv_path") || return 1 + suffix=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_kv_common_suffix") || return 1 + + # 2. Authenticate: GCE identity JWT -> Vault token. + jwt=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + --data-urlencode "audience=http://vault/${role}" \ + --data-urlencode "format=full" \ + "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity") || return 1 + token=$(curl -sf --retry 5 --retry-connrefused \ + --data "$(printf '{"role":"%s","jwt":"%s"}' "$role" "$jwt")" \ + "${addr}/v1/${mount}/login" | jq -re .auth.client_token) || return 1 + + # 3. Read the shared secret blob. + data=$(curl -sf --retry 5 --retry-connrefused --header "X-Vault-Token: ${token}" \ + "${addr}/v1/${kv}/node/${suffix}" | jq -ce .data.data) || return 1 + + # 4. Extract each variable and validate it against its expected format. + METRICS_FLASHBOTS_WORKSPACE=$(echo "$data" | jq -re .METRICS_FLASHBOTS_WORKSPACE) || return 1 + echo "$METRICS_FLASHBOTS_WORKSPACE" | grep -qE '^ws-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' \ + || { echo "vault_fetch: WORKSPACE is not an AMP workspace id" >&2; return 1; } + + METRICS_FLASHBOTS_REGION=$(echo "$data" | jq -re .METRICS_FLASHBOTS_REGION) || return 1 + echo "$METRICS_FLASHBOTS_REGION" | grep -qE '^[a-z]{2}-[a-z]+-[0-9]+$' \ + || { echo "vault_fetch: REGION is not an AWS region" >&2; return 1; } + + METRICS_FLASHBOTS_ACCESS_KEY=$(echo "$data" | jq -re .METRICS_FLASHBOTS_ACCESS_KEY) || return 1 + echo "$METRICS_FLASHBOTS_ACCESS_KEY" | grep -qE '^[A-Z0-9]{20}$' \ + || { echo "vault_fetch: ACCESS_KEY is not an AWS access key id" >&2; return 1; } + + METRICS_FLASHBOTS_SECRET_KEY=$(echo "$data" | jq -re .METRICS_FLASHBOTS_SECRET_KEY) || return 1 + echo "$METRICS_FLASHBOTS_SECRET_KEY" | grep -qE '^[A-Za-z0-9/+]{40}$' \ + || { echo "vault_fetch: SECRET_KEY is not an AWS secret key" >&2; return 1; } + + # 5. All present and well-formed — publish to the environment for envsubst. + export METRICS_FLASHBOTS_WORKSPACE METRICS_FLASHBOTS_REGION \ + METRICS_FLASHBOTS_ACCESS_KEY METRICS_FLASHBOTS_SECRET_KEY +} diff --git a/modules/flashbox/observability/mkosi.postinst b/modules/flashbox/observability/mkosi.postinst new file mode 100755 index 00000000..ec872aa6 --- /dev/null +++ b/modules/flashbox/observability/mkosi.postinst @@ -0,0 +1,12 @@ +#!/bin/bash +set -euxo pipefail + +# Ensure prometheus owns its data directory +mkosi-chroot chown -R prometheus:prometheus /var/lib/prometheus + +# Enable observability services +mkosi-chroot systemctl add-wants minimal.target \ + flashbox-observability-setup.service \ + prometheus.service \ + node-exporter.service \ + process-exporter.service From 16f419f6e8548957f7ebc29ee65a4dc2eb43d475 Mon Sep 17 00:00:00 2001 From: Pablo <118397961+pablin-10@users.noreply.github.com> Date: Wed, 10 Jun 2026 10:09:34 -0300 Subject: [PATCH 03/17] Update readme for flashbox-l1 with GCP pricing (#159) --- modules/flashbox/flashbox-l1/readme.md | 43 +++++++++++++------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index b5eec6fc..ae8a02f8 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -100,34 +100,33 @@ iptables only covers ipv4. For security purposes, we block ipv6 with a kernel fl Machine Specs and Cost ------------------------ -Currently, we deploy Azure’s [DCesv5-series Confidential VMs](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/general-purpose/dcesv5-series?tabs=sizebasic). Unfortunately, these are expensive. For reference, Flashbots production TDX builders run in [Standard_EC32es_v5](https://buildernet.org/docs/operating-a-node#microsoft-azure-cloud) with 32 vCPUs and 2TB Disk, which is $2600/month. [Egress](https://azure.microsoft.com/en-us/pricing/details/bandwidth/) (data transferred out of Azure data centers) costs ~$0.087/GB, and historically this costs TEE searchers $150/month. +We deploy GCP [Confidential VMs](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview) on the **c3-standard** machine series (Intel Sapphire Rapids) with Intel TDX support, in **`us-east4` (Northern Virginia)** to colocate with builders. Lighthouse (consensus) runs on the host, while the searcher runs their own execution client (modified Geth or Reth) and bot inside the container. In the future, we hope to add bare metal support, which will lower this cost dramatically. -We place searcher machines in Azure US East 2 to colocate with builders. - -**Machine** -| Name | CPU | Mem (GB) | Price (USD) | -|------------|-----|----------|-------------| -| DC2es_v5 | 2 | 8 | $70.08 | -| DC4es_v5 | 4 | 16 | $140.16 | -| DC8es_v5 | 8 | 32 | $280.32 | -| DC16es_v5 | 16 | 64 | $560.64 | -| DC32es_v5 | 32 | 128 | $1,121.28 | -| DC48es_v5 | 48 | 192 | $1,681.92 | -| DC64es_v5 | 64 | 256 | $2,242.56 | -| DC96es_v5 | 96 | 384 | $3,363.84 | - -**Disk** -| Size | Price (USD) | -|------|-------------| -| 1TB | $123 | -| 2TB | $235 | -| 4TB | $450 | +**Machine** (base price, VM only) + +`c3-standard-22` is the minimum recommended size. + +| Name | vCPU | Mem (GB) | Price (USD/mo) | +|----------------|------|----------|----------------| +| c3-standard-8 | 8 | 32 | ~$294 | +| c3-standard-22 | 22 | 88 | ~$810 | +| c3-standard-44 | 44 | 176 | ~$1,619 | +| c3-standard-88 | 88 | 352 | ~$3,239 | + +**Disk** (price per TB / month, billed on top of the machine) + +| Type | Price (USD/TB) | Per GB | +|-------------|----------------|-----------| +| pd-balanced | ~$110 | ~$0.11/GB | +| pd-ssd | ~$190 | ~$0.19/GB | + +Total monthly cost is machine + disk — e.g. a `c3-standard-22` with 4 TB pd-balanced ≈ $810 + $440 = **~$1,250/mo**. **Egress** -~$0.087/GB, current TEE searchers pay ~$150/month +Networking/egress and the TDX premium are excluded from the estimates above for now. Attestation Walkthrough ------------------------ From 09abcd1125da923ecdb90ad79b63426c0b0f9f87 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Wed, 10 Jun 2026 15:10:24 +0000 Subject: [PATCH 04/17] Decouple ssh-pubkey-server from the searcher container Order ssh-pubkey-server after dropbear instead of requiring searcher-container, and serve both host keys via the generic --ssh-pubkey-file flag. Generate the dropbear .pub in dropbear-prereq so the control-plane key exists at boot. This makes the attested pubkey channel available before disk init, so the searcher can attest the host key before their first SSH instead of trusting it on first use (TOFU). The container key is served lazily by /pubkey once it appears after the disk is unlocked. Pin ssh-pubkey-server to v0.1.0 (lazy available-subset serving) and update the L1 readme accordingly. --- modules/flashbox/common/mkosi.build | 2 +- .../dropbear.service.d/dropbear-prereq.conf | 3 +++ .../systemd/system/ssh-pubkey-server.service | 11 ++++++++--- modules/flashbox/flashbox-l1/readme.md | 18 +++++++++++------- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/modules/flashbox/common/mkosi.build b/modules/flashbox/common/mkosi.build index 2b0ddbc7..0d3da48d 100755 --- a/modules/flashbox/common/mkosi.build +++ b/modules/flashbox/common/mkosi.build @@ -35,7 +35,7 @@ make_git_package \ # Build ssh-pubkey-server make_git_package \ "ssh-pubkey-server" \ - "multi-key" \ + "v0.1.0" \ "https://github.com/flashbots/ssh-pubkey-server" \ 'go build -trimpath -ldflags "-s -w -buildid= -X github.com/flashbots/go-template/common.Version=v1.0.0" -o ./build/ssh-pubkey-server cmd/httpserver/main.go' \ "build/ssh-pubkey-server:/usr/bin/ssh-pubkey-server" diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf b/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf index cdfcf563..16baa3bc 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf @@ -5,3 +5,6 @@ Requires=wait-for-key.service searcher-firewall.service [Service] ExecStartPre=/usr/bin/chown -R searcher:searcher /home/searcher ExecStartPre=/bin/sh -c 'test -f /etc/dropbear/dropbear_ed25519_host_key || /usr/bin/dropbearkey -t ed25519 -f /etc/dropbear/dropbear_ed25519_host_key' +# Derive the public key so ssh-pubkey-server can serve the control-plane host key +# at boot (before the searcher unlocks the disk), enabling attestation without TOFU. +ExecStartPre=/bin/sh -c 'test -f /etc/dropbear/dropbear_ed25519_host_key.pub || /usr/bin/dropbearkey -y -f /etc/dropbear/dropbear_ed25519_host_key | grep "^ssh-" > /etc/dropbear/dropbear_ed25519_host_key.pub' diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service b/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service index 8159ecbd..d9c012eb 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service @@ -1,11 +1,16 @@ [Unit] Description=SSH Public Key Server -After=searcher-container.service -Requires=searcher-container.service +# Order after dropbear so the host (control-plane) key and its .pub — generated +# by dropbear-prereq — exist before we start. We intentionally do NOT depend on +# searcher-container: that would pin this server behind disk unlock, which +# requires an unattested SSH first (TOFU). The container (data-plane) key is +# served by /pubkey lazily once it appears after the disk is unlocked. +After=dropbear.service +Wants=dropbear.service [Service] Type=simple -ExecStart=/usr/bin/ssh-pubkey-server --listen-addr=127.0.0.1:5001 --ssh-pubkey-file=/etc/searcher/ssh_hostkey/host_key.pub --ssh-pubkey-file=/etc/dropbear/dropbear_ed25519_host_key.pub +ExecStart=/usr/bin/ssh-pubkey-server --listen-addr=127.0.0.1:5001 --ssh-pubkey-file=/etc/dropbear/dropbear_ed25519_host_key.pub --ssh-pubkey-file=/etc/searcher/ssh_hostkey/host_key.pub Restart=always RestartSec=5 diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index b5eec6fc..3a95e42a 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -184,7 +184,7 @@ During container startup, OpenSSH is installed and the SSH key is copied from `e **Searcher Disk Encryption** -On the first startup, after the searcher's SSH key is received and stored, the searcher must SSH into the machine and run the `initialize` command to encrypt their disk. +On the first startup, after the searcher's SSH key is received and stored, the searcher must SSH into the machine and run the `initialize` command to encrypt their disk. Before this first SSH, complete the [Attestation Walkthrough](#attestation-walkthrough) and bind the host key served at `/pubkey` — the attested channel is up at boot, so this initial connection is verified rather than trust-on-first-use. `Tdx-init` prompts the searcher for a passphrase via stdin, [formats]((https://github.com/flashbots/tdx-init/blob/c357e1b5d9bc386c3446e87bddb6dd53ac01ea97/passphrase.go#L43)) the disk with LUKS2 encryption using this passphrase, and [embeds]((https://github.com/flashbots/tdx-init/blob/c357e1b5d9bc386c3446e87bddb6dd53ac01ea97/passphrase.go#L77)) the searcher's SSH key as metadata in the LUKS header. @@ -301,9 +301,13 @@ make build-proxy-client # To trigger remote attestation, open a new terminal and run this command: curl http://127.0.0.1:8080 -# Bind the expected openssh server pubkey to the attested machine IP -# This command ensures that the ssh server the searcher is connecting to -# is indeed the ssh server that is running on the attested machine. +# Bind the attested host keys to known_hosts. +# This ensures the ssh server the searcher connects to is the one running on the +# attested machine. The attested :8745 channel and the host (dropbear) control-plane +# key are available at boot — before you run `initialize` — so the very first SSH +# is verified rather than trust-on-first-use. /pubkey returns whatever host keys +# are currently available; the container (data-plane) key joins once the disk is +# unlocked and the container is up, so re-run this then to bind it too. git clone https://github.com/flashbots/ssh-pubkey-server ./ssh-pubkey-server/cmd/cli/add_to_known_hosts.sh \ @@ -325,7 +329,7 @@ git clone https://github.com/flashbots/ssh-pubkey-server time=2025-07-23T14:00:41.956Z level=INFO msg="Successfully validated attestation document" service=proxy-client version=v0.1.7-1-g4e175a4 time=2025-07-23T14:00:42.051Z level=INFO msg="[proxy-request] proxying complete" service=proxy-client version=v0.1.7-1-g4e175a4 duration=1.275184102s - # fetch openssh server pubkey + # fetch the available host pubkey(s) — the dropbear key is served at boot, before `initialize` ubuntu@schmangeLina-bob-mkosi-builder::~$ curl --insecure https://20.57.71.148:8745/pubkey ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIYZkgqUokLPpIENJPhJAdpNTecgp/1R1RE6XMsIp6Rt @@ -608,8 +612,8 @@ Developer Notes 6. Write new text in `bob.log` to the log socket (**name:** searcher-log-writer.service) (**after:** searcher-log-reader.service) 7. Lighthouse (**name:** `lighthouse.service`) (**after:** `/persistent` is mounted) 8. Start the podman container (**name:** `searcher-container.service`) (**after:** `dropbear.service`, `lighthouse.service`, `searcher-firewall.service`, `/persistent` is mounted) -9. SSH pubkey server (**name:** `ssh-pubkey-server.service`) (**after:** `searcher-container.service`) -10. CVM reverse proxy for SSH pubkey server (**name:** `cvm-reverse-proxy.service`) (**after:** `ssh-pubkey-server.service`) +9. SSH pubkey server (**name:** `ssh-pubkey-server.service`) (**after:** `dropbear.service`) — starts at boot and no longer waits for `searcher-container.service`, so `/pubkey` serves the host (dropbear) key before disk init. The container key is served by `/pubkey` lazily once the container writes it. +10. CVM reverse proxy for SSH pubkey server (**name:** `cvm-reverse-proxy.service`) (**after:** `ssh-pubkey-server.service`) — consequently the attested `:8745` channel is also available at boot, before the searcher's first SSH. ### Testing From 7e4eeebe4dd4facb35f1dd5c3ed7e06bae0ac9b4 Mon Sep 17 00:00:00 2001 From: Pablo <118397961+pablin-10@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:19:09 -0300 Subject: [PATCH 05/17] Add measurements format conversion helper (#161) --- scripts/gcp_measurements_to_dcap.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 scripts/gcp_measurements_to_dcap.sh diff --git a/scripts/gcp_measurements_to_dcap.sh b/scripts/gcp_measurements_to_dcap.sh new file mode 100755 index 00000000..2b2fa0f7 --- /dev/null +++ b/scripts/gcp_measurements_to_dcap.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# TEMPORARY: reshape dstack-mr GCP measurements (make measure-gcp -> +# build/gcp_measurements.json) into cvm-reverse-proxy's dcap-tdx +# --server-measurements format. Drop once cvm-reverse-proxy accepts the +# condensed dstack-mr output directly. +# +# dstack-mr emits mrtd[] (one per known GCP firmware) and rtmr0[] (equal-size +# per-firmware chunks: firmware x machine-type-ACPI x boot variants), while +# cvm-reverse-proxy wants flat measurement sets it ORs over. We pair mrtd[i] +# with its rtmr0 chunk; a genuine quote matches exactly one set. +# +# Usage: scripts/gcp_measurements_to_dcap.sh [gcp_measurements.json] > out.json +set -euo pipefail +jq ' + (.mrtd | length) as $m + | (((.rtmr0 | length) / $m) | floor) as $c + | [ range(0; $m) as $i + | range(0; $c) as $j + | { measurement_id: "local fw\($i)-v\($j)", + attestation_type: "dcap-tdx", + measurements: { + "0": { expected: .mrtd[$i] }, + "1": { expected: .rtmr0[$i*$c + $j] }, + "2": { expected: .rtmr1 }, + "3": { expected: .rtmr2 }, + "4": { expected: .rtmr3 } + } } ] +' "${1:-build/gcp_measurements.json}" From edea6da19b9747ed328a07906f8d9c1a619112ea Mon Sep 17 00:00:00 2001 From: peg Date: Mon, 15 Jun 2026 11:22:23 +0200 Subject: [PATCH 06/17] Use attested-tls-proxy rather than cvm-reverse-proxy --- modules/flashbox/common/mkosi.build | 13 +++++++------ modules/flashbox/common/mkosi.conf | 2 ++ ...rse-proxy.service => attested-tls-proxy.service} | 9 +++++---- 3 files changed, 14 insertions(+), 10 deletions(-) rename modules/flashbox/common/mkosi.extra/etc/systemd/system/{cvm-reverse-proxy.service => attested-tls-proxy.service} (50%) diff --git a/modules/flashbox/common/mkosi.build b/modules/flashbox/common/mkosi.build index 0d3da48d..997f1737 100755 --- a/modules/flashbox/common/mkosi.build +++ b/modules/flashbox/common/mkosi.build @@ -40,12 +40,13 @@ make_git_package \ 'go build -trimpath -ldflags "-s -w -buildid= -X github.com/flashbots/go-template/common.Version=v1.0.0" -o ./build/ssh-pubkey-server cmd/httpserver/main.go' \ "build/ssh-pubkey-server:/usr/bin/ssh-pubkey-server" -make_git_package \ - "cvm-reverse-proxy" \ - "v0.1.8" \ - "https://github.com/flashbots/cvm-reverse-proxy" \ - "make build-proxy-server" \ - "build/proxy-server:/usr/bin/cvm-reverse-proxy" + +# Install attested-tls-proxy +VERSION="v1.1.2" +EXPECTED_SHA256=3b1b1868a5fcb1c0a0b07cfa32a76d2c609c99b9705596cc4a528659208d0349 +curl -sSfL https://github.com/flashbots/attested-tls-proxy/releases/download/${VERSION}/attested-tls-proxy_1.${VERSION}_amd64.deb \ + -o $PACKAGEDIR/attested-tls-proxy.deb +echo "${EXPECTED_SHA256}" $PACKAGEDIR/attested-tls-proxy.deb | sha256sum --check # Build input-only-proxy build_rust_package \ diff --git a/modules/flashbox/common/mkosi.conf b/modules/flashbox/common/mkosi.conf index bf0d19c9..0aad0062 100644 --- a/modules/flashbox/common/mkosi.conf +++ b/modules/flashbox/common/mkosi.conf @@ -8,6 +8,8 @@ PostInstallationScripts=modules/flashbox/common/unmask-systemd.sh modules/flashbox/common/mkosi.postinst BuildScripts=modules/flashbox/common/mkosi.build +VolatilePackages=attested-tls-proxy + Packages=podman catatonit runc diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service similarity index 50% rename from modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service rename to modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service index d7dcc7fe..89ec8e49 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service @@ -1,14 +1,15 @@ [Unit] -Description=SSH Public Key Server +Description=Attested TLS Proxy Server After=ssh-pubkey-server.service Requires=ssh-pubkey-server.service [Service] Type=simple -ExecStart=cvm-reverse-proxy --listen-addr=0.0.0.0:8745 \ - --target-addr=http://localhost:5001 \ +ExecStart=attested-tls-proxy server \ + --inner-listen-addr=0.0.0.0:8745 \ --server-attestation-type=auto \ - --override-azurev6-tcbinfo + --allowed-remote-attestation-type none \ + 127.0.0.1:5001 Restart=always RestartSec=5 From ea0ee537cc93a47b349fbb43b8bee71c408f03be Mon Sep 17 00:00:00 2001 From: peg Date: Mon, 15 Jun 2026 11:32:06 +0200 Subject: [PATCH 07/17] Use 2.0.0 pre-release of attested-tls-proxy --- modules/flashbox/common/mkosi.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/flashbox/common/mkosi.build b/modules/flashbox/common/mkosi.build index 997f1737..f4fef763 100755 --- a/modules/flashbox/common/mkosi.build +++ b/modules/flashbox/common/mkosi.build @@ -42,8 +42,8 @@ make_git_package \ # Install attested-tls-proxy -VERSION="v1.1.2" -EXPECTED_SHA256=3b1b1868a5fcb1c0a0b07cfa32a76d2c609c99b9705596cc4a528659208d0349 +VERSION="v2.0.0-rc.1" +EXPECTED_SHA256=d059daa4a6a2d55a47a69ce35d81fac51fbb2571b92b131169b694cb395a163c curl -sSfL https://github.com/flashbots/attested-tls-proxy/releases/download/${VERSION}/attested-tls-proxy_1.${VERSION}_amd64.deb \ -o $PACKAGEDIR/attested-tls-proxy.deb echo "${EXPECTED_SHA256}" $PACKAGEDIR/attested-tls-proxy.deb | sha256sum --check From d8458c1beeee80d3af023c883a72b5c877a4e3cc Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Tue, 16 Jun 2026 12:57:40 +0000 Subject: [PATCH 08/17] flashbox-l1: allowlist BuilderNet bottom-of-block endpoints (#163) * flashbox-l1: allowlist BuilderNet bottom-of-block endpoints (production only) Add a temporary egress allowlist for the new BuilderNet bottom-of-block endpoints so a searcher can connect to a specific pre-configured builder instance for testing. These 6 IPs serve both the tx stream (persistent wss, path /bob) and bundle submission (bare path) over the same HTTPS endpoint, so they cannot be split into an always-on bundle rule + production-only tx stream rule the way the Flashbots/Titan endpoints are. Since tx stream is an observable side channel, the whole set is gated to production mode. - firewall-config: add 6 BuilderNet IPs to PRODUCTION_OUT (HTTPS/443) - toggle-config: add the same IPs to PRODUCTION_ENDPOINTS so tx-stream flows are torn down via conntrack when leaving production mode - searcher-container-after-init: add static /etc/hosts entries (US nodes first) so the searcher connects by hostname and TLS validates without DNS Tradeoff: BuilderNet bundle submission only works in production mode, unlike the Flashbots/Titan bundle endpoints which are always on. * flashbox-l1: reword BuilderNet endpoint comments to be purely descriptive Drop the "temporary test allowlist" framing and other non-descriptive notes from the BuilderNet egress comments across firewall-config, toggle-config, and searcher-container-after-init. The comments now describe only what the config is and why it is shaped this way. --- .../mkosi.extra/etc/bob/firewall-config | 22 +++++++++++++++++++ .../etc/bob/searcher-container-after-init | 11 ++++++++++ .../mkosi.extra/etc/bob/toggle-config | 9 ++++++++ 3 files changed, 42 insertions(+) diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index 84e95993..3be04540 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -32,6 +32,25 @@ FLASHBOTS_BUNDLE_2="3.15.88.156" FLASHBOTS_TX_STREAM_1="3.136.107.142" FLASHBOTS_TX_STREAM_2="3.149.14.12" +# BuilderNet endpoints (production only) +# +# These IPs serve BOTH the tx stream (persistent wss on path /bob) and bundle +# submission (bare path) over the same HTTPS endpoint, so they are not split into +# an always-on bundle rule + production-only tx stream rule the way the +# Flashbots/Titan endpoints are. Because tx stream is an observable side channel, +# the whole set is gated to production mode and torn down on toggle (see +# toggle-config PRODUCTION_ENDPOINTS). Consequently BuilderNet bundle submission +# works only in production mode, unlike the always-on Flashbots/Titan bundles. +# +# Production mode has no DNS, so these are addressed by IP; the hostnames searchers +# connect to (for TLS) are mapped statically in searcher-container-after-init. +BUILDERNET_FB_USE4="34.85.211.230" # fb-gcp-use4-00 (US east) +BUILDERNET_FB_USE1="200.225.47.181" # fb-om-use1-00 (US east) +BUILDERNET_NM_USE="34.150.234.122" # nethermind-gcp-eastus-101 (US east) +BUILDERNET_FB_EUW4="34.12.189.183" # fb-gcp-euw4-00 (EU west) +BUILDERNET_NM_EUW="34.91.129.156" # nethermind-gcp-westeurope (EU west) +BUILDERNET_FB_ANE1="34.104.157.101" # fb-gcp-ane1-00 (AP northeast) + # Prometheus metrics proxy PROMETHEUS_PROXY_IP="10.88.0.100" @@ -119,3 +138,6 @@ accept_dst_port $CHAIN_MAINTENANCE_OUT udp $EL_P2P_PORT "EL P2P (UDP)" accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $TITAN_IP $TITAN_STATE_DIFF_PORT "Titan state diff WSS" accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 $HTTPS_PORT "Flashbots Protect tx stream" + +# BuilderNet tx stream (path /bob) + bundle submission (bare path), same HTTPS endpoint +accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $BUILDERNET_FB_USE4,$BUILDERNET_FB_USE1,$BUILDERNET_NM_USE,$BUILDERNET_FB_EUW4,$BUILDERNET_NM_EUW,$BUILDERNET_FB_ANE1 $HTTPS_PORT "BuilderNet tx stream + bundle submission" diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init index 2da5acab..c8e976ac 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init @@ -9,4 +9,15 @@ exec_in_container ' 18.221.59.61 backruns.tee-searcher.flashbots.net 3.15.88.156 backruns.tee-searcher.flashbots.net 52.207.17.217 fbtee.titanbuilder.xyz + +# BuilderNet. Searchers connect by hostname so TLS validates (production mode has +# no DNS). US nodes are listed first for latency. Mapping all IPs to +# rpc.buildernet.org here means failover is dial-level (next IP on connect +# failure), not health-aware like the BuilderNet load balancer. +34.85.211.230 fb-gcp-use4-00.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +200.225.47.181 fb-om-use1-00.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +34.150.234.122 nethermind-gcp-eastus-101.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +34.12.189.183 fb-gcp-euw4-00.nodes.buildernet.org direct-eu.buildernet.org rpc.buildernet.org +34.91.129.156 nethermind-gcp-westeurope-100.nodes.buildernet.org direct-eu.buildernet.org rpc.buildernet.org +34.104.157.101 fb-gcp-ane1-00.nodes.buildernet.org direct-ap.buildernet.org rpc.buildernet.org EOF' diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config index 086e17c8..e9fc3265 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config @@ -6,10 +6,19 @@ IMAGE_TYPE="bob-l1" # Production endpoints (format: "IP:Description") +# Established flows to these IPs are killed via conntrack when leaving production +# mode, so the tx-stream side channel does not survive into maintenance mode. PRODUCTION_ENDPOINTS=( "52.207.17.217:Titan state diff" "3.136.107.142:Flashbots protect tx" "3.149.14.12:Flashbots protect tx" + # BuilderNet (tx stream + bundle submission share these IPs) + "34.85.211.230:BuilderNet fb-gcp-use4-00" + "200.225.47.181:BuilderNet fb-om-use1-00" + "34.150.234.122:BuilderNet nethermind-eastus" + "34.12.189.183:BuilderNet fb-gcp-euw4-00" + "34.91.129.156:BuilderNet nethermind-westeurope" + "34.104.157.101:BuilderNet fb-gcp-ane1-00" ) # Maintenance ports (format: "protocol:port:description") From d006fa22423e3634b1ad063f3634b0c843206deb Mon Sep 17 00:00:00 2001 From: Alex Hulbert Date: Wed, 17 Jun 2026 02:26:56 -0400 Subject: [PATCH 09/17] Bump gcp measure script version (#164) --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 0b7443ef..5f699e32 100644 --- a/flake.nix +++ b/flake.nix @@ -44,8 +44,8 @@ src = pkgs.fetchFromGitHub { owner = "flashbots"; repo = "dstack-mr-gcp"; - rev = "ed23e96785ebfb1ff153503b01cfbfb10cffae67"; - sha256 = "sha256-jrHcAEp4OYmOMTJ1BWIULoqKycqqQBWIRLjhKmiZor4="; + rev = "b16e08b32b3dc8f1af7087e12f9970dc91a0b9a0"; + sha256 = "sha256-3KIKgWsDzmLXuRK9YVxX2zJ6jAlZSmRm/bLYE1kJY7k="; }; vendorHash = "sha256-glOyRTrIF/zP78XGV+v58a1Bec6C3Fvc5c8G3PglzPM="; }; From 11d403fcf8d3b4927f12cb690f6f16826d2dfeae Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Wed, 17 Jun 2026 14:39:57 +0000 Subject: [PATCH 10/17] flashbox-l1: drop BuilderNet endpoints in maintenance mode (#165) BuilderNet IPs are allowlisted in PRODUCTION_OUT and torn down on toggle via toggle-config PRODUCTION_ENDPOINTS, but were missing the MAINTENANCE_OUT drop that the Flashbots tx-stream endpoints already have. Because MAINTENANCE_OUT accepts HTTPS to any IP (catch-all), the BuilderNet endpoints -- which carry the tx-stream side channel on 443 -- were reachable in maintenance mode, contradicting their production-only intent. Add the drop before the accept-all rules, mirroring the Flashbots Protect tx-stream treatment. flashbox-l2 reviewed: it has no BuilderNet endpoints (production-only egress is the simulator, already dropped in maintenance), so no change needed there. --- .../flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index 3be04540..abd4992f 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -116,6 +116,9 @@ accept_dst_port $CHAIN_MAINTENANCE_IN udp $EL_P2P_PORT "EL P2P (UDP)" # Block Flashbots protect tx endpoints during maintenance drop_dst_ip $CHAIN_MAINTENANCE_OUT $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 "Flashbots Protect (DROP before accept-all rules)" +# Block BuilderNet endpoints during maintenance. +drop_dst_ip $CHAIN_MAINTENANCE_OUT $BUILDERNET_FB_USE4,$BUILDERNET_FB_USE1,$BUILDERNET_NM_USE,$BUILDERNET_FB_EUW4,$BUILDERNET_NM_EUW,$BUILDERNET_FB_ANE1 "BuilderNet (DROP before accept-all rules)" + accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)" accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)" accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_OVER_TLS_PORT "DNS-over-TLS" From d894f1afafd490755a21a88ea770f944e1143250 Mon Sep 17 00:00:00 2001 From: peg Date: Fri, 19 Jun 2026 11:59:49 +0200 Subject: [PATCH 11/17] Use latest release of attested-tls-proxy --- modules/flashbox/common/mkosi.build | 4 ++-- .../etc/systemd/system/attested-tls-proxy.service | 2 +- modules/flashbox/flashbox-l1/readme.md | 7 ++++--- modules/flashbox/observability/mkosi.postinst | 2 +- scripts/make_git_package.sh | 4 ++-- shared/mkosi.conf | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/flashbox/common/mkosi.build b/modules/flashbox/common/mkosi.build index f4fef763..75ad36e1 100755 --- a/modules/flashbox/common/mkosi.build +++ b/modules/flashbox/common/mkosi.build @@ -42,8 +42,8 @@ make_git_package \ # Install attested-tls-proxy -VERSION="v2.0.0-rc.1" -EXPECTED_SHA256=d059daa4a6a2d55a47a69ce35d81fac51fbb2571b92b131169b694cb395a163c +VERSION="v1.1.3" +EXPECTED_SHA256=3d3f43203bd51be399fe90ab5c633716cbecba0cb30beff291052b748610b65b curl -sSfL https://github.com/flashbots/attested-tls-proxy/releases/download/${VERSION}/attested-tls-proxy_1.${VERSION}_amd64.deb \ -o $PACKAGEDIR/attested-tls-proxy.deb echo "${EXPECTED_SHA256}" $PACKAGEDIR/attested-tls-proxy.deb | sha256sum --check diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service index 89ec8e49..748fc097 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service @@ -6,7 +6,7 @@ Requires=ssh-pubkey-server.service [Service] Type=simple ExecStart=attested-tls-proxy server \ - --inner-listen-addr=0.0.0.0:8745 \ + --listen-addr=0.0.0.0:8745 \ --server-attestation-type=auto \ --allowed-remote-attestation-type none \ 127.0.0.1:5001 diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index 96723038..d16bd7c0 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -285,9 +285,10 @@ Flashbots again leverages Edgeless Constellation’s [attested TLS](https://docs ```bash # download remote attestation tool -git clone https://github.com/flashbots/cvm-reverse-proxy.git -cd cvm-reverse-proxy -make build-proxy-client +git clone https://github.com/flashbots/attested-tls-proxy.git +cd attested-tls-proxy +cargo run -- attested-get --allow-self-signed --measurements-file ./measurements.json --url-path pubkey :8745 +TODO # This will run the client proxy that is listening on port 8080 # and use the server reverse proxy on the deployed image as a target, diff --git a/modules/flashbox/observability/mkosi.postinst b/modules/flashbox/observability/mkosi.postinst index ec872aa6..c32a35f3 100755 --- a/modules/flashbox/observability/mkosi.postinst +++ b/modules/flashbox/observability/mkosi.postinst @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euxo pipefail # Ensure prometheus owns its data directory diff --git a/scripts/make_git_package.sh b/scripts/make_git_package.sh index 2a32a766..f35cb17c 100644 --- a/scripts/make_git_package.sh +++ b/scripts/make_git_package.sh @@ -43,7 +43,7 @@ make_git_package() { mkdir -p "$DESTDIR$dest" cp -r "$cache_dir/$src"/* "$DESTDIR$dest/" else - cp "$cache_dir/$src" "$DESTDIR$dest" + install -m 755 "$cache_dir/$src" "$DESTDIR$dest" fi done return 0 @@ -66,7 +66,7 @@ make_git_package() { cp -r "$build_dir/$src"/* "$DESTDIR$dest/" else mkdir -p "$(dirname "$DESTDIR$dest")" - cp "$build_dir/$src" "$DESTDIR$dest" + install -m 755 "$build_dir/$src" "$DESTDIR$dest" fi # Cache artifact diff --git a/shared/mkosi.conf b/shared/mkosi.conf index 1d594ed2..1b5faa49 100644 --- a/shared/mkosi.conf +++ b/shared/mkosi.conf @@ -19,7 +19,7 @@ Seed=630b5f72-a36a-4e83-b23d-6ef47c82fd9c [Content] SourceDateEpoch=0 -KernelCommandLine=console=tty0 console=ttyS0,115200n8 mitigations=auto,nosmt spec_store_bypass_disable=on nospectre_v2 transparent_hugepage=madvise systemd.unit=minimal.target +KernelCommandLine=console=tty0 console=ttyS0,115200n8 mitigations=auto,nosmt spec_store_bypass_disable=on nospectre_v2 transparent_hugepage=madvise systemd.unit=minimal.target systemd.journald.forward_to_console=yes ExtraTrees=shared/mkosi.extra BuildScripts=shared/mkosi.build.d/* SyncScripts=shared/mkosi.sync.d/* From 757c8324dfef1c196c46f9e28c3f46c904fae693 Mon Sep 17 00:00:00 2001 From: peg Date: Mon, 22 Jun 2026 09:27:26 +0200 Subject: [PATCH 12/17] Update firewall port naming and readme --- .../mkosi.extra/etc/bob/firewall-config | 6 +- modules/flashbox/flashbox-l1/readme.md | 61 +++++++++++-------- .../mkosi.extra/etc/bob/firewall-config | 6 +- scripts/gcp_measurements_to_dcap.sh | 6 +- 4 files changed, 45 insertions(+), 34 deletions(-) diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index 84e95993..f6a49b6b 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -5,7 +5,7 @@ SSH_CONTROL_PORT=22 SSH_DATA_PORT=10022 SSH_REGISTER_PORT=8080 -CVM_REVERSE_PROXY_PORT=8745 +ATTESTED_TLS_PROXY_PORT=8745 SEARCHER_INPUT_UDP_PORT=27017 SEARCHER_INPUT_TCP_PORT=27018 @@ -43,10 +43,10 @@ accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port" accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_UDP_PORT "Searcher UDP input channel" accept_dst_port $CHAIN_ALWAYS_IN tcp $SEARCHER_INPUT_TCP_PORT "Searcher TCP input channel (input-only-proxy)" -# CVM reverse-proxy serves server attestation +# attested-tls-proxy serves server attestation # Also forwards request to ssh pubkey server on localhost:5001, # which serves searcher-container openssh server pubkey -accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy" +accept_dst_port $CHAIN_ALWAYS_IN tcp $ATTESTED_TLS_PROXY_PORT "attested-tls-proxy" # Note: this is CL running on the host accept_dst_port $CHAIN_ALWAYS_IN tcp $CL_P2P_PORT "CL P2P (TCP)" diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index d16bd7c0..2f42e36d 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -78,7 +78,7 @@ Firewall Rules | 54 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | | 80 | Output | HTTP | HTTP | TCP | DISABLED | ENABLED | | 443 | Output | HTTPS | HTTPS | TCP | DISABLED | ENABLED | -| 8745 | Input | CVM-Reverse-Proxy | Host | TCP | ENABLED | ENABLED | +| 8745 | Input | attested-tls-proxy | Host | TCP | ENABLED | ENABLED | | 123 | Output | NTP | Host | UDP | ENABLED | ENABLED | **Searcher Network Namespace iptables** @@ -281,22 +281,22 @@ Then, copy and paste PCR 4, 9, and 11 into the following format and save as `mea ### 3. audit and run the remote attestation software which requests the measurement from Azure’s vTPM -Flashbots again leverages Edgeless Constellation’s [attested TLS](https://docs.edgeless.systems/constellation/architecture/attestation#attested-tls-atls) and other attestation primitives to interact with Azure’s attestation service. CVM-reverse-proxy fetches Azure's vTPM measurement and compares it with the locally supplied measurement. +Flashbots again leverages Edgeless Constellation’s [attested TLS](https://docs.edgeless.systems/constellation/architecture/attestation#attested-tls-atls) and other attestation primitives to interact with Azure’s attestation service. attested-tls-proxy fetches Azure's vTPM measurement and compares it with the locally supplied measurement. ```bash # download remote attestation tool git clone https://github.com/flashbots/attested-tls-proxy.git cd attested-tls-proxy -cargo run -- attested-get --allow-self-signed --measurements-file ./measurements.json --url-path pubkey :8745 -TODO # This will run the client proxy that is listening on port 8080 # and use the server reverse proxy on the deployed image as a target, # marshalling the measurements.json for validation of the attestation. -./cvm-reverse-proxy/build/proxy-client \ ---server-measurements ./measurements.json \ ---target-addr=https://:8745 \ ---log-debug=false +cargo run -- client \ + --listen-addr 127.0.0.1:8080 \ + --allow-self-signed \ + --measurements-file ./measurements.json \ + --log-debug \ + :8745 # To trigger remote attestation, open a new terminal and run this command: curl http://127.0.0.1:8080 @@ -311,7 +311,7 @@ curl http://127.0.0.1:8080 git clone https://github.com/flashbots/ssh-pubkey-server ./ssh-pubkey-server/cmd/cli/add_to_known_hosts.sh \ -./cvm-reverse-proxy/build/proxy-client \ +http://127.0.0.1:8080 \ ``` @@ -319,24 +319,35 @@ git clone https://github.com/flashbots/ssh-pubkey-server Example Output ```bash - # successful attestation - ubuntu@schmangeLina-bob-mkosi-builder:~$ ./cvm-reverse-proxy/build/proxy-client \ - --server-measurements ./measurements.json \ - --target-addr=https://20.57.71.148:8745 \ - --log-debug=false - time=2025-07-23T14:00:33.436Z level=INFO msg="Starting proxy client" service=proxy-client version=v0.1.7-1-g4e175a4 listenAddr=127.0.0.1:8080 - time=2025-07-23T14:00:41.224Z level=INFO msg="Validating attestation document" service=proxy-client version=v0.1.7-1-g4e175a4 - time=2025-07-23T14:00:41.956Z level=INFO msg="Successfully validated attestation document" service=proxy-client version=v0.1.7-1-g4e175a4 - time=2025-07-23T14:00:42.051Z level=INFO msg="[proxy-request] proxying complete" service=proxy-client version=v0.1.7-1-g4e175a4 duration=1.275184102s - - # fetch the available host pubkey(s) — the dropbear key is served at boot, before `initialize` - ubuntu@schmangeLina-bob-mkosi-builder::~$ curl --insecure https://20.57.71.148:8745/pubkey - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIYZkgqUokLPpIENJPhJAdpNTecgp/1R1RE6XMsIp6Rt - + # Start the proxy client + $ cargo run -- client --listen-addr 127.0.0.1:8080 --allow-self-signed --measurements-file ./measurements.json --log-debug 35.255.95.67:8745 + Compiling attested-tls-proxy v1.1.1 (/home/pumkin/src/flashbots/attested-tls-proxy) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.17s + Running `target/debug/attested-tls-proxy client --listen-addr '127.0.0.1:8080' --allow-self-signed --allowed-remote-attestation-type gcp-tdx --log-debug '35.255.95.67:8745'` + 2026-06-22T07:07:38.553942Z DEBUG attested_tls_proxy: [proxy-client] Connected to proxy server with measurements: Some(DCAP({MRTD: "feb7486608382c1ff0e15b4648ddc0acea6ca974eb53e3529f4c4bd5ffbaa20bf335cb75965cea65fe473aed9647c162", RTMR0: "e1d0235496f93f9475bf0b26d33da5c15831cfc94104d6bea7ab82db027c5f1e917d47dda6953eefae7dcb20ab6f75c4", RTMR1: "4ea5a990afef023f89e11fc32d99103d0adc91d5734664542eb980cdabc88224e1fd206d1d3b2eda71f713fdf8308a2b", RTMR2: "c42ba4fb83f99e4b90bc2a3aa9a2e81c5ac578e9a439c23b457ff7dd0d0eae958760616eff05d827289e47608e757547", RTMR3: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"})) + at src/lib.rs:674 + + 2026-06-22T07:07:38.554036Z DEBUG attested_tls_proxy::http_version: [client] Negotiated ALPN Some("flashbots-ratls/1+h2"), chosen protocol Http2 + at src/http_version.rs:39 + + 2026-06-22T07:07:44.356260Z DEBUG attested_tls_proxy: proxy-client accepted connection + at src/lib.rs:594 + + 2026-06-22T07:07:44.356578Z DEBUG attested_tls_proxy: [proxy-client] Read incoming request from source client: Request { method: GET, uri: /pubkey, version: HTTP/1.1, headers: {"host": "127.0.0.1:8080", "user-agent": "curl/8.19.0", "accept": "*/*"}, body: Body(Empty) } + at src/lib.rs:494 + + 2026-06-22T07:07:44.639534Z DEBUG attested_tls_proxy: [proxy-client] Read response from proxy-server: Response { status: 200, version: HTTP/2.0, headers: {"date": "Mon, 22 Jun 2026 07:07:44 GMT", "content-length": "80", "content-type": "text/plain; charset=utf-8"}, body: Body(Streaming) } + at src/lib.rs:498 + + $ curl http://127.0.0.1:8080/pubkey | less + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 80 100 80 0 0 280 0 0 + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHqPgQoc6yLlMsRUvEeV+6oUsvCMV1sr+b1uKTLzu4e9 ``` -If cvm-reverse-proxy returns `Successfully validated attestation document`, the searcher has now verified that they SSH into a genuine TDX VM, running the exact same image as the one they audited locally. In doing so, the searcher has also verified that no one else has access to the container or host, and they can safely upload your arbitrage bot inside ✨🚀 +If attested-tls-proxy client is able to successfully make a connection to the proxy server, the searcher has now verified that they SSH into a genuine TDX VM, running the exact same image as the one they audited locally. In doing so, the searcher has also verified that no one else has access to the container or host, and they can safely upload your arbitrage bot inside ✨🚀 Order Flow APIs ------------------------ @@ -613,7 +624,7 @@ Developer Notes 7. Lighthouse (**name:** `lighthouse.service`) (**after:** `/persistent` is mounted) 8. Start the podman container (**name:** `searcher-container.service`) (**after:** `dropbear.service`, `lighthouse.service`, `searcher-firewall.service`, `/persistent` is mounted) 9. SSH pubkey server (**name:** `ssh-pubkey-server.service`) (**after:** `dropbear.service`) — starts at boot and no longer waits for `searcher-container.service`, so `/pubkey` serves the host (dropbear) key before disk init. The container key is served by `/pubkey` lazily once the container writes it. -10. CVM reverse proxy for SSH pubkey server (**name:** `cvm-reverse-proxy.service`) (**after:** `ssh-pubkey-server.service`) — consequently the attested `:8745` channel is also available at boot, before the searcher's first SSH. +10. Attested TLS proxy for SSH pubkey server (**name:** `attested-tls-proxy.service`) (**after:** `ssh-pubkey-server.service`) — consequently the attested `:8745` channel is also available at boot, before the searcher's first SSH. ### Testing diff --git a/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config index bee194c2..95bc5f53 100644 --- a/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config @@ -5,7 +5,7 @@ SSH_CONTROL_PORT=22 SSH_DATA_PORT=10022 SSH_REGISTER_PORT=8080 -CVM_REVERSE_PROXY_PORT=8745 +ATTESTED_TLS_PROXY_PORT=8745 SEARCHER_INPUT_UDP_PORT=27017 SEARCHER_INPUT_TCP_PORT=27018 @@ -36,10 +36,10 @@ accept_dst_port $CHAIN_ALWAYS_IN tcp $SEARCHER_INPUT_TCP_PORT "Searcher TCP inpu # We assume here that static peers in config are only syn nodes accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$CONFIG_EL_PEERS_IPS" $ENGINE_API_PORT "Engine API" -# CVM reverse-proxy serves server attestation +# attested-tls-proxy serves server attestation # Also forwards request to ssh pubkey server on localhost:5001, # which serves searcher-container openssh server pubkey -accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy" +accept_dst_port $CHAIN_ALWAYS_IN tcp $ATTESTED_TLS_PROXY_PORT "attested-tls-proxy" ########################################################################### # (2) ALWAYS_OUT: Outbound rules that are always applied diff --git a/scripts/gcp_measurements_to_dcap.sh b/scripts/gcp_measurements_to_dcap.sh index 2b2fa0f7..ac68435c 100755 --- a/scripts/gcp_measurements_to_dcap.sh +++ b/scripts/gcp_measurements_to_dcap.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # TEMPORARY: reshape dstack-mr GCP measurements (make measure-gcp -> -# build/gcp_measurements.json) into cvm-reverse-proxy's dcap-tdx -# --server-measurements format. Drop once cvm-reverse-proxy accepts the +# build/gcp_measurements.json) into attestd-tls-proxy's dcap-tdx +# --measurements-file format. Drop once attested-tls-proxy accepts the # condensed dstack-mr output directly. # # dstack-mr emits mrtd[] (one per known GCP firmware) and rtmr0[] (equal-size # per-firmware chunks: firmware x machine-type-ACPI x boot variants), while -# cvm-reverse-proxy wants flat measurement sets it ORs over. We pair mrtd[i] +# attested-tls-proxy wants flat measurement sets it ORs over. We pair mrtd[i] # with its rtmr0 chunk; a genuine quote matches exactly one set. # # Usage: scripts/gcp_measurements_to_dcap.sh [gcp_measurements.json] > out.json From 80be581ebd07b9e68c01b509eab67cbb308ba898 Mon Sep 17 00:00:00 2001 From: peg Date: Mon, 22 Jun 2026 09:44:55 +0200 Subject: [PATCH 13/17] Revert change to mkosi.conf used for testing --- shared/mkosi.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/mkosi.conf b/shared/mkosi.conf index 1b5faa49..1d594ed2 100644 --- a/shared/mkosi.conf +++ b/shared/mkosi.conf @@ -19,7 +19,7 @@ Seed=630b5f72-a36a-4e83-b23d-6ef47c82fd9c [Content] SourceDateEpoch=0 -KernelCommandLine=console=tty0 console=ttyS0,115200n8 mitigations=auto,nosmt spec_store_bypass_disable=on nospectre_v2 transparent_hugepage=madvise systemd.unit=minimal.target systemd.journald.forward_to_console=yes +KernelCommandLine=console=tty0 console=ttyS0,115200n8 mitigations=auto,nosmt spec_store_bypass_disable=on nospectre_v2 transparent_hugepage=madvise systemd.unit=minimal.target ExtraTrees=shared/mkosi.extra BuildScripts=shared/mkosi.build.d/* SyncScripts=shared/mkosi.sync.d/* From 7a17c13aad0429d777aa54978fe93ea827657a26 Mon Sep 17 00:00:00 2001 From: peg Date: Mon, 22 Jun 2026 10:58:29 +0200 Subject: [PATCH 14/17] Typo, tidy readme --- modules/flashbox/flashbox-l1/readme.md | 2 +- scripts/gcp_measurements_to_dcap.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index 2f42e36d..bea4a1cc 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -323,7 +323,7 @@ http://127.0.0.1:8080 \ $ cargo run -- client --listen-addr 127.0.0.1:8080 --allow-self-signed --measurements-file ./measurements.json --log-debug 35.255.95.67:8745 Compiling attested-tls-proxy v1.1.1 (/home/pumkin/src/flashbots/attested-tls-proxy) Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.17s - Running `target/debug/attested-tls-proxy client --listen-addr '127.0.0.1:8080' --allow-self-signed --allowed-remote-attestation-type gcp-tdx --log-debug '35.255.95.67:8745'` + Running `target/debug/attested-tls-proxy client --listen-addr '127.0.0.1:8080' --allow-self-signed --measurements-file ./measurements.json --log-debug '35.255.95.67:8745'` 2026-06-22T07:07:38.553942Z DEBUG attested_tls_proxy: [proxy-client] Connected to proxy server with measurements: Some(DCAP({MRTD: "feb7486608382c1ff0e15b4648ddc0acea6ca974eb53e3529f4c4bd5ffbaa20bf335cb75965cea65fe473aed9647c162", RTMR0: "e1d0235496f93f9475bf0b26d33da5c15831cfc94104d6bea7ab82db027c5f1e917d47dda6953eefae7dcb20ab6f75c4", RTMR1: "4ea5a990afef023f89e11fc32d99103d0adc91d5734664542eb980cdabc88224e1fd206d1d3b2eda71f713fdf8308a2b", RTMR2: "c42ba4fb83f99e4b90bc2a3aa9a2e81c5ac578e9a439c23b457ff7dd0d0eae958760616eff05d827289e47608e757547", RTMR3: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"})) at src/lib.rs:674 diff --git a/scripts/gcp_measurements_to_dcap.sh b/scripts/gcp_measurements_to_dcap.sh index ac68435c..a498a169 100755 --- a/scripts/gcp_measurements_to_dcap.sh +++ b/scripts/gcp_measurements_to_dcap.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # TEMPORARY: reshape dstack-mr GCP measurements (make measure-gcp -> -# build/gcp_measurements.json) into attestd-tls-proxy's dcap-tdx +# build/gcp_measurements.json) into attested-tls-proxy's dcap-tdx # --measurements-file format. Drop once attested-tls-proxy accepts the # condensed dstack-mr output directly. # From 52ebc50f9f246e32ee15829c7f924ffb39e86da2 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Tue, 23 Jun 2026 19:29:33 +0000 Subject: [PATCH 15/17] flashbox-l1: document BuilderNet bottom-of-block in readme (#166) * flashbox-l1: document BuilderNet bottom-of-block in readme BuilderNet bottom-of-block order flow is now live, with the same API format as Titan's. Document it alongside the existing Flashbots/Titan order-flow sections. - Add "Searching on BuilderNet's Bottom of Block" section: state diff stream (eth_subscribe / flashbots_stateDiffs) on wss://rpc.buildernet.org/bob and bundle submission (eth_sendBobBundle) on https://rpc.buildernet.org. Format mirrors Titan; both share one HTTPS endpoint so both are production-only. - Add the section to the table of contents. - Add a BuilderNet row to the firewall rules table (443, production only). - List BuilderNet in the intro line of supported order flow. - Fix DNS row port in the firewall table (54 -> 53). * Added note on readme --------- Co-authored-by: pablin-10 <118397961+pablin-10@users.noreply.github.com> --- modules/flashbox/flashbox-l1/readme.md | 139 ++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 2 deletions(-) diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index 96723038..395f3c0b 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -1,7 +1,7 @@ TEE Searcher === -Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect and Titan Builder's bottom of block. +Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect, Titan Builder's bottom of block, and BuilderNet's bottom of block. - [TDX Mental Model](#tdx-mental-model) - [Image Overview](#image-overview) @@ -11,6 +11,7 @@ Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun - [Order Flow APIs](#order-flow-apis) - [Flashbots Protect](#searching-on-flashbots-protect-transactions) - [Titan Builder](#searching-on-titan-builders-bottom-of-block) + - [BuilderNet](#searching-on-buildernets-bottom-of-block) - [Disk Persistence](#disk-persistence) - [Searcher Commands and Services](#searcher-commands-and-services) - [Developer Notes](#developer-notes) @@ -73,9 +74,10 @@ Firewall Rules | 9000 | Input + Output | Consensus Client P2P | Podman | TCP + UDP | ENABLED | ENABLED | | 443 | Output **IP WHITELISTED** | Flashbots Protect Tx Stream | Podman | TCP | ENABLED | DISABLED | | 42203 | Output **IP WHITELISTED** | Titan Builder State Diff Stream | Podman | TCP | ENABLED | DISABLED | +| 443 | Output **IP WHITELISTED** | BuilderNet State Diff Stream + Bundle RPC | BuilderNet RPC | TCP | ENABLED | DISABLED | | 443 | Output **IP WHITELISTED** | Flashbots Bundle RPC | Flashbots Bundle RPC | TCP | ENABLED | ENABLED | | 1338 | Output **IP WHITELISTED** | Titan Bundle RPC | Titan Bundle RPC | TCP | ENABLED | ENABLED | -| 54 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | +| 53 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | | 80 | Output | HTTP | HTTP | TCP | DISABLED | ENABLED | | 443 | Output | HTTPS | HTTPS | TCP | DISABLED | ENABLED | | 8745 | Input | CVM-Reverse-Proxy | Host | TCP | ENABLED | ENABLED | @@ -493,6 +495,139 @@ Note on `targetPools`: - Titan Builder will use `targetPools` to determine what other blocks to consider adding the bottom of block bundle to. - Searchers should include the address of the contract that’s state change causes the arbitrage. For example, the Uni V2 pool address. +### Searching on BuilderNet's Bottom of Block + +BuilderNet serves both the state diff stream and bundle submission over the same HTTPS endpoint (`rpc.buildernet.org`). Because they share one endpoint, both are reachable only in **production mode** — unlike the Flashbots and Titan bundle RPCs, which are always on. + +**Subscribing to BuilderNet's State Diff Stream** + +**Connecting** + +Connect to the server located at: +``` +wss://rpc.buildernet.org/bob +``` + +Use the `eth_subscribe` method to subscribe to state diffs: + +```json +{"method":"eth_subscribe","params":["flashbots_stateDiffs"]} +``` + +**Response** + +```json +{ + "jsonrpc": "2.0", + "result": "whzoOReHirSJxxF8Z0bqvbghmXjD3hWRW0", + "id": 1 +} +``` + +You'll start receiving state diffs: + +``` +{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": { + "subscription": "whzoOReHirSJxxF8Z0bqvbghmXjD3hWRW0", + "result": { + "blockNumber": "String", // hex encoded block number the block builder is currently building for + "blockTimestamp": "String", // hex encoded seconds since the unix epoch + "blockUuid": "String", // a UUID V4 that is used to identify the current block being streamed + "stateOverrides": "Object" { // a nested object of changed addresses to changed storage slot keys and their updated value + "address": { + "balance": "String" + "code": "String" // ONLY IF CONTRACT IS DEPLOYED IN THIS BLOCK + "nonce": "String" + "stateDiff": { + "": "String" + } + } + } + } + } +} +``` +
+Example Output + + ```json + 2024-12-03 23:46:27,370 - __main__ - INFO - Initializing WebSocket connection to ws://127.0.0.1:8547 + 2024-12-03 23:46:27,375 - __main__ - INFO - Subscribed to state diffs + 2024-12-03 23:46:27,377 - __main__ - INFO - Subscription response: {"jsonrpc":"2.0","result":"aYF2ehyZ8I4fz3rxkkRiOxtnfFqWosI9HC","id":1} + 2024-12-03 23:46:33,108 - __main__ - INFO - Parsed state diff: { + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": { + "subscription": "aYF2ehyZ8I4fz3rxkkRiOxtnfFqWosI9HC", + "result": { + "blockNumber": "0x1456624", + "blockTimestamp": "0x674f985b", + "blockUuid": "b3041804-c0ff-4628-9581-29910f78593e", + "stateOverrides": { + "0x0000000000a39bb272e79075ade125fd351887ac": { + "balance": "0x35c9406dfc78d4448d9", + "nonce": "0x1", + "stateDiff": { + "0xffc5f4bf805d0f20d7ba2d180bf4492e98716db6def51fb60972294b5ba556cf": "0x0000000000000000000000000000000000000000000000000905438e60010000" + } + }, + "0x111111111117dc0aa78b770fa6a738034120c302": { + "stateDiff": { + "0xc0ec8fbf02d70b2873f5a76f503e97bd1b0ca8048ab517fad231214a74ebe459": "0x0000000000000000000000000000000000000000000ebc80f5e0cbee39cca338", + "0xcb4547a880ed764ae6e3838e74f0795915d3b91357e4852c97ce6e0cdcf6c023": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "0x1a44076050125825900e736c501f859c50fe728c": { + "stateDiff": { + "0xe988aa870f58bb597aadbc090e6f5508b7e93c0ad1d3effac7f5825387d9975e": "0x05626211c42f691c213286fd1cc93859d96b1e96d1e919cd2738013a25857823" + } + }, + "0x9355d11cb5c6e8a301d131c5ee1c7fdc032dbb9a": { + "balance": "0x0", + "code": "0x363d3d373d3d3d363d735397d0869aba0d55e96d5716d383f6e1d8695ed75af43d82803e903d91602b57fd5bf3000000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x1", + "stateDiff": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000101679fb19dec9d66c34450a8563ffdfd29c04e615a" + } + } + } + } + } + } + ``` +
+ + +**Sending Bottom of Block Bundles to BuilderNet RPC** + +Connect to the server located at: +``` +https://rpc.buildernet.org +``` + +Use the `eth_sendBobBundle` method to submit bundles: + +``` +{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_sendBobBundle", + "params": [ + { // regular eth_sendBundle fields + txs, + blockNumber, + }, + targetUuid, // String, block UUID that this bundle is targeting eg 123e4567-e89b-12d3-a456-426614174000 + targetPools // Array[String], A list of pool addresses that this bundle is targeting + ] +} +``` + +Note: `targetUuid` and `targetPools` are currently accepted but ignored by BuilderNet. When support for them is added, their placement in the request may differ from what is shown above. + Disk Persistence ------------------------ From 2e3a88ca8032c799bbfd1c587a8c01e002117c07 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Tue, 23 Jun 2026 19:31:39 +0000 Subject: [PATCH 16/17] flashbox-l1: remove Titan Builder bottom-of-block endpoints (#167) Titan Builder's bottom of block is no longer supported, so remove its endpoints from the image config and documentation entirely. Code (modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/): - firewall-config: drop TITAN_* vars, the ALWAYS_OUT Titan bundle rule, and the PRODUCTION_OUT Titan state diff rule; update BuilderNet comments that compared against "Flashbots/Titan" to just "Flashbots". - toggle-config: drop the Titan entry from PRODUCTION_ENDPOINTS. - searcher-container-after-init: drop the fbtee.titanbuilder.xyz /etc/hosts entry. Docs (readme.md): - Remove the "Searching on Titan Builder's Bottom of Block" section, its TOC entry, the two Titan firewall-table rows, and the intro-line mention; drop the comparative Titan mention from the BuilderNet section. --- .../mkosi.extra/etc/bob/firewall-config | 17 +-- .../etc/bob/searcher-container-after-init | 1 - .../mkosi.extra/etc/bob/toggle-config | 1 - modules/flashbox/flashbox-l1/readme.md | 140 +----------------- 4 files changed, 4 insertions(+), 155 deletions(-) diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index abd4992f..7f0e7953 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -19,12 +19,6 @@ NTP_NTS_PORT=4460 CL_P2P_PORT=9000 EL_P2P_PORT=30303 -TITAN_IP="52.207.17.217" -TITAN_BUNDLE_PORT=1338 # bundle submission (always on) -TITAN_STATE_DIFF_PORT=42203 # state diff stream (production only) -# If Titan state diff port ever changes to any of accepted in maintenance, -# update MAINTENANCE_OUT the same way as for FLASHBOTS_TX_STREAM* - # Flashbots bundle (always on) FLASHBOTS_BUNDLE_1="18.221.59.61" FLASHBOTS_BUNDLE_2="3.15.88.156" @@ -37,10 +31,10 @@ FLASHBOTS_TX_STREAM_2="3.149.14.12" # These IPs serve BOTH the tx stream (persistent wss on path /bob) and bundle # submission (bare path) over the same HTTPS endpoint, so they are not split into # an always-on bundle rule + production-only tx stream rule the way the -# Flashbots/Titan endpoints are. Because tx stream is an observable side channel, +# Flashbots endpoints are. Because tx stream is an observable side channel, # the whole set is gated to production mode and torn down on toggle (see # toggle-config PRODUCTION_ENDPOINTS). Consequently BuilderNet bundle submission -# works only in production mode, unlike the always-on Flashbots/Titan bundles. +# works only in production mode, unlike the always-on Flashbots bundles. # # Production mode has no DNS, so these are addressed by IP; the hostnames searchers # connect to (for TLS) are mapped statically in searcher-container-after-init. @@ -84,12 +78,6 @@ accept_dst_port $CHAIN_ALWAYS_OUT udp $CL_P2P_PORT "CL P2P (UDP)" accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP" accept_dst_port $CHAIN_ALWAYS_OUT tcp $NTP_NTS_PORT "NTP-NTS" -# Titan builder bundle endpoints (always on) -# Security note: This is a side channel. -# While the operator will not be able to see the content of the packets, -# they can observe the presence or absence of packets. -accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $TITAN_IP $TITAN_BUNDLE_PORT "Titan builder bundle" - # Flashbots bundle endpoints (always on) # Security note: This is a side channel. # While the operator will not be able to see the content of the packets, @@ -139,7 +127,6 @@ accept_dst_port $CHAIN_MAINTENANCE_OUT udp $EL_P2P_PORT "EL P2P (UDP)" # (6) PRODUCTION_OUT: Outbound rules for Production Mode ########################################################################### -accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $TITAN_IP $TITAN_STATE_DIFF_PORT "Titan state diff WSS" accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 $HTTPS_PORT "Flashbots Protect tx stream" # BuilderNet tx stream (path /bob) + bundle submission (bare path), same HTTPS endpoint diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init index c8e976ac..761ed603 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init @@ -8,7 +8,6 @@ exec_in_container ' 3.136.107.142 tx.tee-searcher.flashbots.net 18.221.59.61 backruns.tee-searcher.flashbots.net 3.15.88.156 backruns.tee-searcher.flashbots.net -52.207.17.217 fbtee.titanbuilder.xyz # BuilderNet. Searchers connect by hostname so TLS validates (production mode has # no DNS). US nodes are listed first for latency. Mapping all IPs to diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config index e9fc3265..a60eb48a 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config @@ -9,7 +9,6 @@ IMAGE_TYPE="bob-l1" # Established flows to these IPs are killed via conntrack when leaving production # mode, so the tx-stream side channel does not survive into maintenance mode. PRODUCTION_ENDPOINTS=( - "52.207.17.217:Titan state diff" "3.136.107.142:Flashbots protect tx" "3.149.14.12:Flashbots protect tx" # BuilderNet (tx stream + bundle submission share these IPs) diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index 395f3c0b..1d958525 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -1,7 +1,7 @@ TEE Searcher === -Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect, Titan Builder's bottom of block, and BuilderNet's bottom of block. +Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect and BuilderNet's bottom of block. - [TDX Mental Model](#tdx-mental-model) - [Image Overview](#image-overview) @@ -10,7 +10,6 @@ Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun - [Attestation Walkthrough](#attestation-walkthrough) - [Order Flow APIs](#order-flow-apis) - [Flashbots Protect](#searching-on-flashbots-protect-transactions) - - [Titan Builder](#searching-on-titan-builders-bottom-of-block) - [BuilderNet](#searching-on-buildernets-bottom-of-block) - [Disk Persistence](#disk-persistence) - [Searcher Commands and Services](#searcher-commands-and-services) @@ -73,10 +72,8 @@ Firewall Rules | 30303 | Input + Output | Execution Client P2P | Podman | TCP + UDP | DISABLED | ENABLED | | 9000 | Input + Output | Consensus Client P2P | Podman | TCP + UDP | ENABLED | ENABLED | | 443 | Output **IP WHITELISTED** | Flashbots Protect Tx Stream | Podman | TCP | ENABLED | DISABLED | -| 42203 | Output **IP WHITELISTED** | Titan Builder State Diff Stream | Podman | TCP | ENABLED | DISABLED | | 443 | Output **IP WHITELISTED** | BuilderNet State Diff Stream + Bundle RPC | BuilderNet RPC | TCP | ENABLED | DISABLED | | 443 | Output **IP WHITELISTED** | Flashbots Bundle RPC | Flashbots Bundle RPC | TCP | ENABLED | ENABLED | -| 1338 | Output **IP WHITELISTED** | Titan Bundle RPC | Titan Bundle RPC | TCP | ENABLED | ENABLED | | 53 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | | 80 | Output | HTTP | HTTP | TCP | DISABLED | ENABLED | | 443 | Output | HTTPS | HTTPS | TCP | DISABLED | ENABLED | @@ -362,142 +359,9 @@ To submit bundles, connect to the server: https://backruns.tee-searcher.flashbots.net ``` -### Searching on Titan Builder's Bottom of Block - -**Subscribing to Titan's State Diff Stream** - -**Connecting** - -Connect to the server located at: -``` -wss://fbtee.titanbuilder.xyz:42203 -``` - -Use the `eth_subscribe` method to subscribe to state diffs: - -```json -{"method":"eth_subscribe","params":["flashbots_stateDiffs"]} -``` - -**Response** - -```json -{ - "jsonrpc": "2.0", - "result": "whzoOReHirSJxxF8Z0bqvbghmXjD3hWRW0", - "id": 1 -} -``` - -You'll start receiving state diffs: - -``` -{ - "jsonrpc": "2.0", - "method": "eth_subscription", - "params": { - "subscription": "whzoOReHirSJxxF8Z0bqvbghmXjD3hWRW0", - "result": { - "blockNumber": "String", // hex encoded block number the block builder is currently building for - "blockTimestamp": "String", // hex encoded seconds since the unix epoch - "blockUuid": "String", // a UUID V4 that is used to identify the current block being streamed - "stateOverrides": "Object" { // a nested object of changed addresses to changed storage slot keys and their updated value - "address": { - "balance": "String" - "code": "String" // ONLY IF CONTRACT IS DEPLOYED IN THIS BLOCK - "nonce": "String" - "stateDiff": { - "": "String" - } - } - } - } - } -} -``` -
-Example Output - - ```json - 2024-12-03 23:46:27,370 - __main__ - INFO - Initializing WebSocket connection to ws://127.0.0.1:8547 - 2024-12-03 23:46:27,375 - __main__ - INFO - Subscribed to state diffs - 2024-12-03 23:46:27,377 - __main__ - INFO - Subscription response: {"jsonrpc":"2.0","result":"aYF2ehyZ8I4fz3rxkkRiOxtnfFqWosI9HC","id":1} - 2024-12-03 23:46:33,108 - __main__ - INFO - Parsed state diff: { - "jsonrpc": "2.0", - "method": "eth_subscription", - "params": { - "subscription": "aYF2ehyZ8I4fz3rxkkRiOxtnfFqWosI9HC", - "result": { - "blockNumber": "0x1456624", - "blockTimestamp": "0x674f985b", - "blockUuid": "b3041804-c0ff-4628-9581-29910f78593e", - "stateOverrides": { - "0x0000000000a39bb272e79075ade125fd351887ac": { - "balance": "0x35c9406dfc78d4448d9", - "nonce": "0x1", - "stateDiff": { - "0xffc5f4bf805d0f20d7ba2d180bf4492e98716db6def51fb60972294b5ba556cf": "0x0000000000000000000000000000000000000000000000000905438e60010000" - } - }, - "0x111111111117dc0aa78b770fa6a738034120c302": { - "stateDiff": { - "0xc0ec8fbf02d70b2873f5a76f503e97bd1b0ca8048ab517fad231214a74ebe459": "0x0000000000000000000000000000000000000000000ebc80f5e0cbee39cca338", - "0xcb4547a880ed764ae6e3838e74f0795915d3b91357e4852c97ce6e0cdcf6c023": "0x0000000000000000000000000000000000000000000000000000000000000000" - } - }, - "0x1a44076050125825900e736c501f859c50fe728c": { - "stateDiff": { - "0xe988aa870f58bb597aadbc090e6f5508b7e93c0ad1d3effac7f5825387d9975e": "0x05626211c42f691c213286fd1cc93859d96b1e96d1e919cd2738013a25857823" - } - }, - "0x9355d11cb5c6e8a301d131c5ee1c7fdc032dbb9a": { - "balance": "0x0", - "code": "0x363d3d373d3d3d363d735397d0869aba0d55e96d5716d383f6e1d8695ed75af43d82803e903d91602b57fd5bf3000000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x1", - "stateDiff": { - "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000101679fb19dec9d66c34450a8563ffdfd29c04e615a" - } - } - } - } - } - } - ``` -
- - -**Sending Bottom of Block Bundles to Titan RPC** - -Connect to the server located at: -``` -https://fbtee.titanbuilder.xyz:1338 -``` - -Use the `eth_sendBobBundle` method to submit bundles: - -``` -{ - "jsonrpc": "2.0", - "id": 1, - "method": "eth_sendBobBundle", - "params": [ - { // regular eth_sendBundle fields - txs, - blockNumber, - }, - targetUuid, // String, block UUID that this bundle is targeting eg 123e4567-e89b-12d3-a456-426614174000 - targetPools // Array[String], A list of pool addresses that this bundle is targeting - ] -} -``` - -Note on `targetPools`: -- Titan Builder will use `targetPools` to determine what other blocks to consider adding the bottom of block bundle to. -- Searchers should include the address of the contract that’s state change causes the arbitrage. For example, the Uni V2 pool address. - ### Searching on BuilderNet's Bottom of Block -BuilderNet serves both the state diff stream and bundle submission over the same HTTPS endpoint (`rpc.buildernet.org`). Because they share one endpoint, both are reachable only in **production mode** — unlike the Flashbots and Titan bundle RPCs, which are always on. +BuilderNet serves both the state diff stream and bundle submission over the same HTTPS endpoint (`rpc.buildernet.org`). Because they share one endpoint, both are reachable only in **production mode** — unlike the Flashbots bundle RPC, which is always on. **Subscribing to BuilderNet's State Diff Stream** From 7bc5e48d0aae12a892c79ec392bfa0ba0e0bc214 Mon Sep 17 00:00:00 2001 From: Pablo <118397961+pablin-10@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:56:54 -0300 Subject: [PATCH 17/17] Add reusable build+publish workflow for flashbox-l1 (#173) * Add reusable build+publish workflow for flashbox-l1 --------- Co-authored-by: Niccolo Raspa --- .../workflows/_build-and-publish-image.yaml | 276 ++++++++++++++++++ .github/workflows/build.yaml | 5 +- .github/workflows/flashbox-l1.yaml | 44 +++ 3 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/_build-and-publish-image.yaml create mode 100644 .github/workflows/flashbox-l1.yaml diff --git a/.github/workflows/_build-and-publish-image.yaml b/.github/workflows/_build-and-publish-image.yaml new file mode 100644 index 00000000..d9ce0263 --- /dev/null +++ b/.github/workflows/_build-and-publish-image.yaml @@ -0,0 +1,276 @@ +# Reusable GitHub Actions workflow for building and publishing mkosi images. +# +# Builds the specified image, uploads the artifacts to R2, and creates a draft +# GitHub Release when the build is driven by a release tag. +# +# Behavior: +# - Release tag (-vX.Y.Z, pushed or passed as `ref`): +# - Builds the image at that tag +# - Uploads artifacts to R2 under -images/vX.Y.Z/ +# - Creates a draft GitHub Release with measurements +# +# - Any other ref (branch, SHA, or empty): +# - Builds a dev image at that ref +# - Uploads artifacts to R2 under -images/dev/./ +# - No release is created + +name: Build and publish image + +on: + workflow_call: + inputs: + image-id: + description: "Image to build (images/.conf)." + type: string + required: true + ref: + description: "Tag, SHA, or branch to build. A release tag selects release mode; anything else is a dev build. Empty = the triggering ref." + type: string + default: "" + dev-image: + description: "Build a dev image (devtools profile) via `make build-dev`. Produces -dev_* artifacts." + type: boolean + default: false + +permissions: + contents: write + +jobs: + build: + name: Build ${{ inputs.image-id }} + runs-on: warp-ubuntu-latest-x64-32x + env: + R2_BUCKET: flashbots-public-artifacts + R2_PUBLIC_BASE_URL: "https://builder-artifacts.flashbots.net" + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + # Use Nix to build image + - name: Bypass Lima wrapper + run: touch .bypass-lima + + - name: Enable user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + # A build is release when an -vX.Y.Z tag is pushed or passed as `ref` + # Everything else is a dev build. + - name: Determine build mode + env: + IMAGE: ${{ inputs.image-id }} + REF_INPUT: ${{ inputs.ref }} + DEV_IMAGE: ${{ inputs.dev-image }} + run: | + set -euo pipefail + TAG_RE="^${IMAGE}-v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$" + + RELEASE_TAG="" + if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF_TYPE}" == "tag" ]]; then + if ! [[ "${GITHUB_REF_NAME}" =~ ${TAG_RE} ]]; then + echo "::error::pushed tag '${GITHUB_REF_NAME}' doesn't match ${IMAGE}-vX.Y.Z[-suffix]" + exit 1 + fi + RELEASE_TAG="${GITHUB_REF_NAME}" + elif [[ "${REF_INPUT}" =~ ${TAG_RE} ]]; then + RELEASE_TAG="${REF_INPUT}" + fi + + if [[ -n "${RELEASE_TAG}" && "${DEV_IMAGE}" == "true" ]]; then + echo "::error::dev-image builds cannot be released" + exit 1 + fi + + # build-dev builds under image id -dev, which changes the + # artifact filenames. + if [[ "${DEV_IMAGE}" == "true" ]]; then + ARTIFACT_PREFIX="${IMAGE}-dev" + else + ARTIFACT_PREFIX="${IMAGE}" + fi + + if [[ -n "${RELEASE_TAG}" ]]; then + VERSION_DIR="v${RELEASE_TAG#"${IMAGE}"-v}" + IMAGE_VERSION="" # mkosi.version supplies the version + else + # GITHUB_SHA differs from the checked-out HEAD when a custom ref is passed. + HEAD_SHA="$(git rev-parse HEAD)" + # Lowercase to [a-z0-9-] so the GCS image name and R2 path stay valid. + BRANCH="$(printf '%s' "${REF_INPUT:-${GITHUB_REF_NAME}}" \ + | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+|-+$//g')" + # env_wrapper.sh's latest.* symlink handling requires a version + # containing a dot. + IMAGE_VERSION="${BRANCH}.${HEAD_SHA:0:8}" + VERSION_DIR="dev/${IMAGE_VERSION}" + fi + + { + echo "IMAGE=${IMAGE}" + echo "ARTIFACT_PREFIX=${ARTIFACT_PREFIX}" + echo "RELEASE_TAG=${RELEASE_TAG}" + echo "R2_PATH_PREFIX=${IMAGE}-images" + echo "VERSION_DIR=${VERSION_DIR}" + echo "IMAGE_VERSION=${IMAGE_VERSION}" + } >> "$GITHUB_ENV" + echo "mode: RELEASE_TAG=${RELEASE_TAG:-}, VERSION_DIR=${VERSION_DIR}" + + # MKOSI_EXTRA_ARGS is picked up and appended to mkosi by env_wrapper.sh. + - name: Build image + env: + DEV_IMAGE: ${{ inputs.dev-image }} + run: | + set -euo pipefail + umask 022 + if [[ -n "${IMAGE_VERSION}" ]]; then + export MKOSI_EXTRA_ARGS="--image-version=${IMAGE_VERSION}" + fi + if [[ "${DEV_IMAGE}" == "true" ]]; then + make build-dev IMAGE="${IMAGE}" + else + make build IMAGE="${IMAGE}" + fi + + - name: Fix permissions + run: sudo chown -R "$(id -u):$(id -g)" build/ + + - name: Export measurements (GCP TDX) + run: | + set -euo pipefail + make measure-gcp + + - name: Show build artifacts + run: ls -lh build/ + + - name: Install rclone + run: | + set -euo pipefail + RCLONE_DEB=rclone-v1.74.2-linux-amd64.deb + curl -fsSL -o "${RCLONE_DEB}" "https://downloads.rclone.org/v1.74.2/${RCLONE_DEB}" + echo "d2232f57e47ff4f9b30f1b575ce95567b1c51f22526c4e98814641dfd835c103 ${RCLONE_DEB}" | sha256sum -c - + sudo apt-get install -y "./${RCLONE_DEB}" + rm -f "${RCLONE_DEB}" + + - name: Upload artifacts to R2 + env: + R2_ACCESS_KEY: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_ACCESS_KEY }} + R2_SECRET_KEY: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_SECRET_KEY }} + R2_ENDPOINT: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_ENDPOINT }} + run: | + set -euo pipefail + mkdir -p ~/.config/rclone + # use_data_integrity_protections=false: R2 returns 501 NotImplemented + # for the CRC32 checksum trailers aws-sdk-go-v2 sends by default. + cat > ~/.config/rclone/rclone.conf < [!NOTE]" + echo "> **\`gcp_measurements.json\`** (attached, also on R2): the expected TDX measurements (MRTD, RTMR0-3) for this image on GCP, from [\`dstack-mr-gcp\`](https://github.com/flashbots/dstack-mr-gcp). Reproduce them yourself with the steps below." + echo ">" + echo "> 📖 Details: [attestation walkthrough](https://github.com/flashbots/flashbots-images/blob/main/modules/flashbox/flashbox-l1/readme.md#attestation-walkthrough) · [cvm-reverse-proxy](https://github.com/flashbots/cvm-reverse-proxy)" + echo "" + echo "### 🔁 Reproduce the measurements" + echo "" + echo "1. Clone and check out this release:" + echo ' ```bash' + echo " git clone ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} && cd ${GITHUB_REPOSITORY#*/}" + echo " git checkout ${RELEASE_TAG}" + echo ' ```' + echo "2. Build the image (~30 min):" + echo ' ```bash' + echo " make build IMAGE=${IMAGE}" + echo ' ```' + echo "3. Reproduce the measurements:" + echo ' ```bash' + echo " make measure-gcp" + echo ' ```' + echo " Your \`build/gcp_measurements.json\` should be identical to the attached one." + echo "" + echo "### 🔐 Attest your running VM" + echo "" + echo "To verify your running VM, generate an Intel-signed TDX quote and validate it against expected measurements using [\`cvm-reverse-proxy\`](https://github.com/flashbots/cvm-reverse-proxy) over attested TLS." + echo "" + echo "1. Set your instance IP and build the client:" + echo ' ```bash' + echo " VM_IP=" + echo " git clone https://github.com/flashbots/cvm-reverse-proxy && make -C cvm-reverse-proxy build-proxy-client" + echo ' ```' + echo "2. Convert your measurements to the proxy's format (temporary [jq shim](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${RELEASE_TAG}/scripts/gcp_measurements_to_dcap.sh)):" + echo ' ```bash' + echo " scripts/gcp_measurements_to_dcap.sh > gcp-server-measurements.json" + echo ' ```' + echo "3. Attest:" + echo ' ```bash' + echo " cvm-reverse-proxy/build/proxy-client --server-measurements gcp-server-measurements.json --target-addr=https://\${VM_IP}:8745 &" + echo " curl -si http://127.0.0.1:8080 | grep -q X-Flashbots-Measurement && echo '✅ VM matches YOUR build' || echo '❌ mismatch'" + echo ' ```' + } > release-notes.md + + # Attach the measurements and the .efi UKI; the larger tar.gz/qcow2 + # artifacts stay in R2 and are linked in the release body. + assets=() + for f in build/gcp_measurements.json build/"${ARTIFACT_PREFIX}"_*.efi; do + [ -e "$f" ] && assets+=("$f") + done + + gh release create "${RELEASE_TAG}" --draft \ + --title "${RELEASE_TAG}" --notes-file release-notes.md "${assets[@]}" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a72f0f87..92080b52 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,7 +13,6 @@ # - Reproducibility test (default: false) # - Adds a second build on a separate runner and compares SHA256 hashes - name: Build mkosi images on: @@ -87,7 +86,7 @@ jobs: name: build ${{ matrix.image }} image runs-on: warp-ubuntu-latest-x64-32x steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ inputs.branch || github.ref }} @@ -150,7 +149,7 @@ jobs: name: reprotest ${{ matrix.image }} runs-on: warp-ubuntu-latest-x64-32x steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ inputs.branch || github.ref }} diff --git a/.github/workflows/flashbox-l1.yaml b/.github/workflows/flashbox-l1.yaml new file mode 100644 index 00000000..bf23837c --- /dev/null +++ b/.github/workflows/flashbox-l1.yaml @@ -0,0 +1,44 @@ +# GitHub Actions workflow for building and publishing the flashbox-l1 image. +# +# Triggers on: +# - Pushes of a `flashbox-l1-v*` tag: builds the image, uploads the artifacts +# to R2, and creates a draft GitHub Release +# +# - Manual dispatch: allows specifying: +# - Ref to build (default: the current branch) +# - Branch or SHA → dev build, uploaded to R2 only (no release) +# - Existing release tag → deterministic rebuild and a new draft Release +# - Dev image (default: false) +# - Builds with the devtools profile (e.g. SSH access); never released +# +# All build and publish logic lives in the reusable workflow +# _build-and-publish-image.yaml; this file only sets the image id. + +name: flashbox-l1 + +on: + push: + tags: + - "flashbox-l1-v*" + workflow_dispatch: + inputs: + ref: + description: "Tag, SHA, or branch to build (empty = this branch)." + type: string + default: "" + dev-image: + description: "Build a dev image (devtools profile)" + type: boolean + default: false + +jobs: + release: + permissions: + contents: write + uses: ./.github/workflows/_build-and-publish-image.yaml + with: + image-id: flashbox-l1 + # Inputs are empty on tag pushes; fall back to the defaults. + ref: ${{ inputs.ref || '' }} + dev-image: ${{ inputs.dev-image || false }} + secrets: inherit