diff --git a/.github/workflows/_build-and-publish-image.yaml b/.github/workflows/_build-and-publish-image.yaml new file mode 100644 index 00000000..d9ce0263 --- /dev/null +++ b/.github/workflows/_build-and-publish-image.yaml @@ -0,0 +1,276 @@ +# Reusable GitHub Actions workflow for building and publishing mkosi images. +# +# Builds the specified image, uploads the artifacts to R2, and creates a draft +# GitHub Release when the build is driven by a release tag. +# +# Behavior: +# - Release tag (-vX.Y.Z, pushed or passed as `ref`): +# - Builds the image at that tag +# - Uploads artifacts to R2 under -images/vX.Y.Z/ +# - Creates a draft GitHub Release with measurements +# +# - Any other ref (branch, SHA, or empty): +# - Builds a dev image at that ref +# - Uploads artifacts to R2 under -images/dev/./ +# - No release is created + +name: Build and publish image + +on: + workflow_call: + inputs: + image-id: + description: "Image to build (images/.conf)." + type: string + required: true + ref: + description: "Tag, SHA, or branch to build. A release tag selects release mode; anything else is a dev build. Empty = the triggering ref." + type: string + default: "" + dev-image: + description: "Build a dev image (devtools profile) via `make build-dev`. Produces -dev_* artifacts." + type: boolean + default: false + +permissions: + contents: write + +jobs: + build: + name: Build ${{ inputs.image-id }} + runs-on: warp-ubuntu-latest-x64-32x + env: + R2_BUCKET: flashbots-public-artifacts + R2_PUBLIC_BASE_URL: "https://builder-artifacts.flashbots.net" + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + # Use Nix to build image + - name: Bypass Lima wrapper + run: touch .bypass-lima + + - name: Enable user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + # A build is release when an -vX.Y.Z tag is pushed or passed as `ref` + # Everything else is a dev build. + - name: Determine build mode + env: + IMAGE: ${{ inputs.image-id }} + REF_INPUT: ${{ inputs.ref }} + DEV_IMAGE: ${{ inputs.dev-image }} + run: | + set -euo pipefail + TAG_RE="^${IMAGE}-v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$" + + RELEASE_TAG="" + if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF_TYPE}" == "tag" ]]; then + if ! [[ "${GITHUB_REF_NAME}" =~ ${TAG_RE} ]]; then + echo "::error::pushed tag '${GITHUB_REF_NAME}' doesn't match ${IMAGE}-vX.Y.Z[-suffix]" + exit 1 + fi + RELEASE_TAG="${GITHUB_REF_NAME}" + elif [[ "${REF_INPUT}" =~ ${TAG_RE} ]]; then + RELEASE_TAG="${REF_INPUT}" + fi + + if [[ -n "${RELEASE_TAG}" && "${DEV_IMAGE}" == "true" ]]; then + echo "::error::dev-image builds cannot be released" + exit 1 + fi + + # build-dev builds under image id -dev, which changes the + # artifact filenames. + if [[ "${DEV_IMAGE}" == "true" ]]; then + ARTIFACT_PREFIX="${IMAGE}-dev" + else + ARTIFACT_PREFIX="${IMAGE}" + fi + + if [[ -n "${RELEASE_TAG}" ]]; then + VERSION_DIR="v${RELEASE_TAG#"${IMAGE}"-v}" + IMAGE_VERSION="" # mkosi.version supplies the version + else + # GITHUB_SHA differs from the checked-out HEAD when a custom ref is passed. + HEAD_SHA="$(git rev-parse HEAD)" + # Lowercase to [a-z0-9-] so the GCS image name and R2 path stay valid. + BRANCH="$(printf '%s' "${REF_INPUT:-${GITHUB_REF_NAME}}" \ + | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+|-+$//g')" + # env_wrapper.sh's latest.* symlink handling requires a version + # containing a dot. + IMAGE_VERSION="${BRANCH}.${HEAD_SHA:0:8}" + VERSION_DIR="dev/${IMAGE_VERSION}" + fi + + { + echo "IMAGE=${IMAGE}" + echo "ARTIFACT_PREFIX=${ARTIFACT_PREFIX}" + echo "RELEASE_TAG=${RELEASE_TAG}" + echo "R2_PATH_PREFIX=${IMAGE}-images" + echo "VERSION_DIR=${VERSION_DIR}" + echo "IMAGE_VERSION=${IMAGE_VERSION}" + } >> "$GITHUB_ENV" + echo "mode: RELEASE_TAG=${RELEASE_TAG:-}, VERSION_DIR=${VERSION_DIR}" + + # MKOSI_EXTRA_ARGS is picked up and appended to mkosi by env_wrapper.sh. + - name: Build image + env: + DEV_IMAGE: ${{ inputs.dev-image }} + run: | + set -euo pipefail + umask 022 + if [[ -n "${IMAGE_VERSION}" ]]; then + export MKOSI_EXTRA_ARGS="--image-version=${IMAGE_VERSION}" + fi + if [[ "${DEV_IMAGE}" == "true" ]]; then + make build-dev IMAGE="${IMAGE}" + else + make build IMAGE="${IMAGE}" + fi + + - name: Fix permissions + run: sudo chown -R "$(id -u):$(id -g)" build/ + + - name: Export measurements (GCP TDX) + run: | + set -euo pipefail + make measure-gcp + + - name: Show build artifacts + run: ls -lh build/ + + - name: Install rclone + run: | + set -euo pipefail + RCLONE_DEB=rclone-v1.74.2-linux-amd64.deb + curl -fsSL -o "${RCLONE_DEB}" "https://downloads.rclone.org/v1.74.2/${RCLONE_DEB}" + echo "d2232f57e47ff4f9b30f1b575ce95567b1c51f22526c4e98814641dfd835c103 ${RCLONE_DEB}" | sha256sum -c - + sudo apt-get install -y "./${RCLONE_DEB}" + rm -f "${RCLONE_DEB}" + + - name: Upload artifacts to R2 + env: + R2_ACCESS_KEY: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_ACCESS_KEY }} + R2_SECRET_KEY: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_SECRET_KEY }} + R2_ENDPOINT: ${{ secrets.R2_FLASHBOTS_PUBLIC_ARTIFACTS_ENDPOINT }} + run: | + set -euo pipefail + mkdir -p ~/.config/rclone + # use_data_integrity_protections=false: R2 returns 501 NotImplemented + # for the CRC32 checksum trailers aws-sdk-go-v2 sends by default. + cat > ~/.config/rclone/rclone.conf < [!NOTE]" + echo "> **\`gcp_measurements.json\`** (attached, also on R2): the expected TDX measurements (MRTD, RTMR0-3) for this image on GCP, from [\`dstack-mr-gcp\`](https://github.com/flashbots/dstack-mr-gcp). Reproduce them yourself with the steps below." + echo ">" + echo "> 📖 Details: [attestation walkthrough](https://github.com/flashbots/flashbots-images/blob/main/modules/flashbox/flashbox-l1/readme.md#attestation-walkthrough) · [cvm-reverse-proxy](https://github.com/flashbots/cvm-reverse-proxy)" + echo "" + echo "### 🔁 Reproduce the measurements" + echo "" + echo "1. Clone and check out this release:" + echo ' ```bash' + echo " git clone ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY} && cd ${GITHUB_REPOSITORY#*/}" + echo " git checkout ${RELEASE_TAG}" + echo ' ```' + echo "2. Build the image (~30 min):" + echo ' ```bash' + echo " make build IMAGE=${IMAGE}" + echo ' ```' + echo "3. Reproduce the measurements:" + echo ' ```bash' + echo " make measure-gcp" + echo ' ```' + echo " Your \`build/gcp_measurements.json\` should be identical to the attached one." + echo "" + echo "### 🔐 Attest your running VM" + echo "" + echo "To verify your running VM, generate an Intel-signed TDX quote and validate it against expected measurements using [\`cvm-reverse-proxy\`](https://github.com/flashbots/cvm-reverse-proxy) over attested TLS." + echo "" + echo "1. Set your instance IP and build the client:" + echo ' ```bash' + echo " VM_IP=" + echo " git clone https://github.com/flashbots/cvm-reverse-proxy && make -C cvm-reverse-proxy build-proxy-client" + echo ' ```' + echo "2. Convert your measurements to the proxy's format (temporary [jq shim](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${RELEASE_TAG}/scripts/gcp_measurements_to_dcap.sh)):" + echo ' ```bash' + echo " scripts/gcp_measurements_to_dcap.sh > gcp-server-measurements.json" + echo ' ```' + echo "3. Attest:" + echo ' ```bash' + echo " cvm-reverse-proxy/build/proxy-client --server-measurements gcp-server-measurements.json --target-addr=https://\${VM_IP}:8745 &" + echo " curl -si http://127.0.0.1:8080 | grep -q X-Flashbots-Measurement && echo '✅ VM matches YOUR build' || echo '❌ mismatch'" + echo ' ```' + } > release-notes.md + + # Attach the measurements and the .efi UKI; the larger tar.gz/qcow2 + # artifacts stay in R2 and are linked in the release body. + assets=() + for f in build/gcp_measurements.json build/"${ARTIFACT_PREFIX}"_*.efi; do + [ -e "$f" ] && assets+=("$f") + done + + gh release create "${RELEASE_TAG}" --draft \ + --title "${RELEASE_TAG}" --notes-file release-notes.md "${assets[@]}" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a72f0f87..92080b52 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,7 +13,6 @@ # - Reproducibility test (default: false) # - Adds a second build on a separate runner and compares SHA256 hashes - name: Build mkosi images on: @@ -87,7 +86,7 @@ jobs: name: build ${{ matrix.image }} image runs-on: warp-ubuntu-latest-x64-32x steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ inputs.branch || github.ref }} @@ -150,7 +149,7 @@ jobs: name: reprotest ${{ matrix.image }} runs-on: warp-ubuntu-latest-x64-32x steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ inputs.branch || github.ref }} diff --git a/.github/workflows/flashbox-l1.yaml b/.github/workflows/flashbox-l1.yaml new file mode 100644 index 00000000..bf23837c --- /dev/null +++ b/.github/workflows/flashbox-l1.yaml @@ -0,0 +1,44 @@ +# GitHub Actions workflow for building and publishing the flashbox-l1 image. +# +# Triggers on: +# - Pushes of a `flashbox-l1-v*` tag: builds the image, uploads the artifacts +# to R2, and creates a draft GitHub Release +# +# - Manual dispatch: allows specifying: +# - Ref to build (default: the current branch) +# - Branch or SHA → dev build, uploaded to R2 only (no release) +# - Existing release tag → deterministic rebuild and a new draft Release +# - Dev image (default: false) +# - Builds with the devtools profile (e.g. SSH access); never released +# +# All build and publish logic lives in the reusable workflow +# _build-and-publish-image.yaml; this file only sets the image id. + +name: flashbox-l1 + +on: + push: + tags: + - "flashbox-l1-v*" + workflow_dispatch: + inputs: + ref: + description: "Tag, SHA, or branch to build (empty = this branch)." + type: string + default: "" + dev-image: + description: "Build a dev image (devtools profile)" + type: boolean + default: false + +jobs: + release: + permissions: + contents: write + uses: ./.github/workflows/_build-and-publish-image.yaml + with: + image-id: flashbox-l1 + # Inputs are empty on tag pushes; fall back to the defaults. + ref: ${{ inputs.ref || '' }} + dev-image: ${{ inputs.dev-image || false }} + secrets: inherit diff --git a/flake.nix b/flake.nix index b5533853..5f699e32 100644 --- a/flake.nix +++ b/flake.nix @@ -44,8 +44,8 @@ src = pkgs.fetchFromGitHub { owner = "flashbots"; repo = "dstack-mr-gcp"; - rev = "ecf3284b72a507fd005de91d49f7372490cf6995"; - sha256 = "sha256-1rcm9sIZuvCojNN2HMPrsECYn9sd8eVChsgwleo8nFY="; + rev = "b16e08b32b3dc8f1af7087e12f9970dc91a0b9a0"; + sha256 = "sha256-3KIKgWsDzmLXuRK9YVxX2zJ6jAlZSmRm/bLYE1kJY7k="; }; vendorHash = "sha256-glOyRTrIF/zP78XGV+v58a1Bec6C3Fvc5c8G3PglzPM="; }; diff --git a/images/flashbox-l1.conf b/images/flashbox-l1.conf index 42f61ce8..a5d07dfd 100644 --- a/images/flashbox-l1.conf +++ b/images/flashbox-l1.conf @@ -2,6 +2,7 @@ Include=shared/mkosi.conf Include=modules/flashbox/common/mkosi.conf Include=modules/flashbox/flashbox-l1/mkosi.conf +Include=modules/flashbox/observability/mkosi.conf [Config] Profiles=azure,gcp diff --git a/modules/flashbox/common/mkosi.build b/modules/flashbox/common/mkosi.build index 2b0ddbc7..75ad36e1 100755 --- a/modules/flashbox/common/mkosi.build +++ b/modules/flashbox/common/mkosi.build @@ -35,17 +35,18 @@ make_git_package \ # Build ssh-pubkey-server make_git_package \ "ssh-pubkey-server" \ - "multi-key" \ + "v0.1.0" \ "https://github.com/flashbots/ssh-pubkey-server" \ 'go build -trimpath -ldflags "-s -w -buildid= -X github.com/flashbots/go-template/common.Version=v1.0.0" -o ./build/ssh-pubkey-server cmd/httpserver/main.go' \ "build/ssh-pubkey-server:/usr/bin/ssh-pubkey-server" -make_git_package \ - "cvm-reverse-proxy" \ - "v0.1.8" \ - "https://github.com/flashbots/cvm-reverse-proxy" \ - "make build-proxy-server" \ - "build/proxy-server:/usr/bin/cvm-reverse-proxy" + +# Install attested-tls-proxy +VERSION="v1.1.3" +EXPECTED_SHA256=3d3f43203bd51be399fe90ab5c633716cbecba0cb30beff291052b748610b65b +curl -sSfL https://github.com/flashbots/attested-tls-proxy/releases/download/${VERSION}/attested-tls-proxy_1.${VERSION}_amd64.deb \ + -o $PACKAGEDIR/attested-tls-proxy.deb +echo "${EXPECTED_SHA256}" $PACKAGEDIR/attested-tls-proxy.deb | sha256sum --check # Build input-only-proxy build_rust_package \ diff --git a/modules/flashbox/common/mkosi.conf b/modules/flashbox/common/mkosi.conf index bf0d19c9..0aad0062 100644 --- a/modules/flashbox/common/mkosi.conf +++ b/modules/flashbox/common/mkosi.conf @@ -8,6 +8,8 @@ PostInstallationScripts=modules/flashbox/common/unmask-systemd.sh modules/flashbox/common/mkosi.postinst BuildScripts=modules/flashbox/common/mkosi.build +VolatilePackages=attested-tls-proxy + Packages=podman catatonit runc diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service similarity index 50% rename from modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service rename to modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service index d7dcc7fe..748fc097 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/cvm-reverse-proxy.service +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/attested-tls-proxy.service @@ -1,14 +1,15 @@ [Unit] -Description=SSH Public Key Server +Description=Attested TLS Proxy Server After=ssh-pubkey-server.service Requires=ssh-pubkey-server.service [Service] Type=simple -ExecStart=cvm-reverse-proxy --listen-addr=0.0.0.0:8745 \ - --target-addr=http://localhost:5001 \ +ExecStart=attested-tls-proxy server \ + --listen-addr=0.0.0.0:8745 \ --server-attestation-type=auto \ - --override-azurev6-tcbinfo + --allowed-remote-attestation-type none \ + 127.0.0.1:5001 Restart=always RestartSec=5 diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf b/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf index cdfcf563..16baa3bc 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/dropbear.service.d/dropbear-prereq.conf @@ -5,3 +5,6 @@ Requires=wait-for-key.service searcher-firewall.service [Service] ExecStartPre=/usr/bin/chown -R searcher:searcher /home/searcher ExecStartPre=/bin/sh -c 'test -f /etc/dropbear/dropbear_ed25519_host_key || /usr/bin/dropbearkey -t ed25519 -f /etc/dropbear/dropbear_ed25519_host_key' +# Derive the public key so ssh-pubkey-server can serve the control-plane host key +# at boot (before the searcher unlocks the disk), enabling attestation without TOFU. +ExecStartPre=/bin/sh -c 'test -f /etc/dropbear/dropbear_ed25519_host_key.pub || /usr/bin/dropbearkey -y -f /etc/dropbear/dropbear_ed25519_host_key | grep "^ssh-" > /etc/dropbear/dropbear_ed25519_host_key.pub' diff --git a/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service b/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service index 8159ecbd..d9c012eb 100644 --- a/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service +++ b/modules/flashbox/common/mkosi.extra/etc/systemd/system/ssh-pubkey-server.service @@ -1,11 +1,16 @@ [Unit] Description=SSH Public Key Server -After=searcher-container.service -Requires=searcher-container.service +# Order after dropbear so the host (control-plane) key and its .pub — generated +# by dropbear-prereq — exist before we start. We intentionally do NOT depend on +# searcher-container: that would pin this server behind disk unlock, which +# requires an unattested SSH first (TOFU). The container (data-plane) key is +# served by /pubkey lazily once it appears after the disk is unlocked. +After=dropbear.service +Wants=dropbear.service [Service] Type=simple -ExecStart=/usr/bin/ssh-pubkey-server --listen-addr=127.0.0.1:5001 --ssh-pubkey-file=/etc/searcher/ssh_hostkey/host_key.pub --ssh-pubkey-file=/etc/dropbear/dropbear_ed25519_host_key.pub +ExecStart=/usr/bin/ssh-pubkey-server --listen-addr=127.0.0.1:5001 --ssh-pubkey-file=/etc/dropbear/dropbear_ed25519_host_key.pub --ssh-pubkey-file=/etc/searcher/ssh_hostkey/host_key.pub Restart=always RestartSec=5 diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh b/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh index b4251905..221c84b8 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh +++ b/modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh @@ -8,6 +8,9 @@ SEARCHER_SSH_PORT=10022 SEARCHER_INPUT_UDP_PORT=27017 # External UDP input channel SEARCHER_INPUT_TCP_PORT=27018 # External TCP input channel +# ENDPOINTS +PROMETHEUS_PROXY_IP=10.88.0.100 # host firewall allows in ALWAYS_OUT + # Run extra commands which are customized per image, # see bob*/mkosi.extra/etc/bob/searcher-container-before-init # @@ -73,6 +76,9 @@ ns_iptables() { ns_iptables -A OUTPUT -d 169.254.169.254 -j DROP +# Block container from reaching the internal Prometheus Proxy +ns_iptables -A OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP + # Block consensus layer P2P port (TCP and UDP) ns_iptables -A OUTPUT -p tcp --dport 9000 -j DROP ns_iptables -A OUTPUT -p udp --dport 9000 -j DROP diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh b/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh index 8701b55e..2e7f3a01 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh +++ b/modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh @@ -151,6 +151,7 @@ drop_dst_ip() { # # `source` is not supported in dash ########################################################################### + . /etc/bob/firewall-config ########################################################################### diff --git a/modules/flashbox/common/mkosi.extra/usr/bin/toggle b/modules/flashbox/common/mkosi.extra/usr/bin/toggle index 088001a7..13f5f0df 100755 --- a/modules/flashbox/common/mkosi.extra/usr/bin/toggle +++ b/modules/flashbox/common/mkosi.extra/usr/bin/toggle @@ -45,6 +45,9 @@ CHAIN_MAINTENANCE_OUT="MAINTENANCE_OUT" CHAIN_PRODUCTION_IN="PRODUCTION_IN" CHAIN_PRODUCTION_OUT="PRODUCTION_OUT" +# Endpoints +PROMETHEUS_PROXY_IP="10.88.0.100" + # Source image-specific configuration if available TOGGLE_CONFIG="/etc/bob/toggle-config" if [ -f "$TOGGLE_CONFIG" ]; then @@ -217,11 +220,13 @@ check_searcher_namespace_rules() { && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27017 -j DROP 2>/dev/null \ && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27018 -j DROP 2>/dev/null \ && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p udp --dport 123 -j DROP 2>/dev/null \ - && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null; then - echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018) are ACTIVE." + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null \ + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d 169.254.169.254 -j DROP 2>/dev/null \ + && nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP 2>/dev/null; then + echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are ACTIVE." return 0 else - echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123) are NOT found." + echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are NOT found." return 1 fi } diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config index 4d88d139..5a50855f 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config @@ -5,7 +5,7 @@ SSH_CONTROL_PORT=22 SSH_DATA_PORT=10022 SSH_REGISTER_PORT=8080 -CVM_REVERSE_PROXY_PORT=8745 +ATTESTED_TLS_PROXY_PORT=8745 SEARCHER_INPUT_UDP_PORT=27017 SEARCHER_INPUT_TCP_PORT=27018 @@ -19,12 +19,6 @@ NTP_NTS_PORT=4460 CL_P2P_PORT=9000 EL_P2P_PORT=30303 -TITAN_IP="52.207.17.217" -TITAN_BUNDLE_PORT=1338 # bundle submission (always on) -TITAN_STATE_DIFF_PORT=42203 # state diff stream (production only) -# If Titan state diff port ever changes to any of accepted in maintenance, -# update MAINTENANCE_OUT the same way as for FLASHBOTS_TX_STREAM* - # Flashbots bundle (always on) FLASHBOTS_BUNDLE_1="18.221.59.61" FLASHBOTS_BUNDLE_2="3.15.88.156" @@ -32,6 +26,28 @@ FLASHBOTS_BUNDLE_2="3.15.88.156" FLASHBOTS_TX_STREAM_1="3.136.107.142" FLASHBOTS_TX_STREAM_2="3.149.14.12" +# BuilderNet endpoints (production only) +# +# These IPs serve BOTH the tx stream (persistent wss on path /bob) and bundle +# submission (bare path) over the same HTTPS endpoint, so they are not split into +# an always-on bundle rule + production-only tx stream rule the way the +# Flashbots endpoints are. Because tx stream is an observable side channel, +# the whole set is gated to production mode and torn down on toggle (see +# toggle-config PRODUCTION_ENDPOINTS). Consequently BuilderNet bundle submission +# works only in production mode, unlike the always-on Flashbots bundles. +# +# Production mode has no DNS, so these are addressed by IP; the hostnames searchers +# connect to (for TLS) are mapped statically in searcher-container-after-init. +BUILDERNET_FB_USE4="34.85.211.230" # fb-gcp-use4-00 (US east) +BUILDERNET_FB_USE1="200.225.47.181" # fb-om-use1-00 (US east) +BUILDERNET_NM_USE="34.150.234.122" # nethermind-gcp-eastus-101 (US east) +BUILDERNET_FB_EUW4="34.12.189.183" # fb-gcp-euw4-00 (EU west) +BUILDERNET_NM_EUW="34.91.129.156" # nethermind-gcp-westeurope (EU west) +BUILDERNET_FB_ANE1="34.104.157.101" # fb-gcp-ane1-00 (AP northeast) + +# Prometheus metrics proxy +PROMETHEUS_PROXY_IP="10.88.0.100" + ########################################################################### # (1) ALWAYS_IN: Inbound rules that are always applied ########################################################################### @@ -40,10 +56,10 @@ accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port" accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_UDP_PORT "Searcher UDP input channel" accept_dst_port $CHAIN_ALWAYS_IN tcp $SEARCHER_INPUT_TCP_PORT "Searcher TCP input channel (input-only-proxy)" -# CVM reverse-proxy serves server attestation +# attested-tls-proxy serves server attestation # Also forwards request to ssh pubkey server on localhost:5001, # which serves searcher-container openssh server pubkey -accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy" +accept_dst_port $CHAIN_ALWAYS_IN tcp $ATTESTED_TLS_PROXY_PORT "attested-tls-proxy" # Note: this is CL running on the host accept_dst_port $CHAIN_ALWAYS_IN tcp $CL_P2P_PORT "CL P2P (TCP)" @@ -62,18 +78,15 @@ accept_dst_port $CHAIN_ALWAYS_OUT udp $CL_P2P_PORT "CL P2P (UDP)" accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP" accept_dst_port $CHAIN_ALWAYS_OUT tcp $NTP_NTS_PORT "NTP-NTS" -# Titan builder bundle endpoints (always on) -# Security note: This is a side channel. -# While the operator will not be able to see the content of the packets, -# they can observe the presence or absence of packets. -accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $TITAN_IP $TITAN_BUNDLE_PORT "Titan builder bundle" - # Flashbots bundle endpoints (always on) # Security note: This is a side channel. # While the operator will not be able to see the content of the packets, # they can observe the presence or absence of packets. accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $FLASHBOTS_BUNDLE_1,$FLASHBOTS_BUNDLE_2 $HTTPS_PORT "Flashbots bundle" +# Prometheus metrics proxy (searcher netns has a DROP on this IP — see init-container.sh) +accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $PROMETHEUS_PROXY_IP $HTTPS_PORT "Prometheus metrics proxy" + ########################################################################### # (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode ########################################################################### @@ -91,6 +104,9 @@ accept_dst_port $CHAIN_MAINTENANCE_IN udp $EL_P2P_PORT "EL P2P (UDP)" # Block Flashbots protect tx endpoints during maintenance drop_dst_ip $CHAIN_MAINTENANCE_OUT $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 "Flashbots Protect (DROP before accept-all rules)" +# Block BuilderNet endpoints during maintenance. +drop_dst_ip $CHAIN_MAINTENANCE_OUT $BUILDERNET_FB_USE4,$BUILDERNET_FB_USE1,$BUILDERNET_NM_USE,$BUILDERNET_FB_EUW4,$BUILDERNET_NM_EUW,$BUILDERNET_FB_ANE1 "BuilderNet (DROP before accept-all rules)" + accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)" accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)" accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_OVER_TLS_PORT "DNS-over-TLS" @@ -111,5 +127,7 @@ accept_dst_port $CHAIN_MAINTENANCE_OUT udp $EL_P2P_PORT "EL P2P (UDP)" # (6) PRODUCTION_OUT: Outbound rules for Production Mode ########################################################################### -accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $TITAN_IP $TITAN_STATE_DIFF_PORT "Titan state diff WSS" accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 $HTTPS_PORT "Flashbots Protect tx stream" + +# BuilderNet tx stream (path /bob) + bundle submission (bare path), same HTTPS endpoint +accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp $BUILDERNET_FB_USE4,$BUILDERNET_FB_USE1,$BUILDERNET_NM_USE,$BUILDERNET_FB_EUW4,$BUILDERNET_NM_EUW,$BUILDERNET_FB_ANE1 $HTTPS_PORT "BuilderNet tx stream + bundle submission" diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init index 2da5acab..761ed603 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/searcher-container-after-init @@ -8,5 +8,15 @@ exec_in_container ' 3.136.107.142 tx.tee-searcher.flashbots.net 18.221.59.61 backruns.tee-searcher.flashbots.net 3.15.88.156 backruns.tee-searcher.flashbots.net -52.207.17.217 fbtee.titanbuilder.xyz + +# BuilderNet. Searchers connect by hostname so TLS validates (production mode has +# no DNS). US nodes are listed first for latency. Mapping all IPs to +# rpc.buildernet.org here means failover is dial-level (next IP on connect +# failure), not health-aware like the BuilderNet load balancer. +34.85.211.230 fb-gcp-use4-00.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +200.225.47.181 fb-om-use1-00.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +34.150.234.122 nethermind-gcp-eastus-101.nodes.buildernet.org direct-us.buildernet.org rpc.buildernet.org +34.12.189.183 fb-gcp-euw4-00.nodes.buildernet.org direct-eu.buildernet.org rpc.buildernet.org +34.91.129.156 nethermind-gcp-westeurope-100.nodes.buildernet.org direct-eu.buildernet.org rpc.buildernet.org +34.104.157.101 fb-gcp-ane1-00.nodes.buildernet.org direct-ap.buildernet.org rpc.buildernet.org EOF' diff --git a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config index 086e17c8..a60eb48a 100644 --- a/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config +++ b/modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/toggle-config @@ -6,10 +6,18 @@ IMAGE_TYPE="bob-l1" # Production endpoints (format: "IP:Description") +# Established flows to these IPs are killed via conntrack when leaving production +# mode, so the tx-stream side channel does not survive into maintenance mode. PRODUCTION_ENDPOINTS=( - "52.207.17.217:Titan state diff" "3.136.107.142:Flashbots protect tx" "3.149.14.12:Flashbots protect tx" + # BuilderNet (tx stream + bundle submission share these IPs) + "34.85.211.230:BuilderNet fb-gcp-use4-00" + "200.225.47.181:BuilderNet fb-om-use1-00" + "34.150.234.122:BuilderNet nethermind-eastus" + "34.12.189.183:BuilderNet fb-gcp-euw4-00" + "34.91.129.156:BuilderNet nethermind-westeurope" + "34.104.157.101:BuilderNet fb-gcp-ane1-00" ) # Maintenance ports (format: "protocol:port:description") diff --git a/modules/flashbox/flashbox-l1/readme.md b/modules/flashbox/flashbox-l1/readme.md index b5eec6fc..aa8cb5c9 100644 --- a/modules/flashbox/flashbox-l1/readme.md +++ b/modules/flashbox/flashbox-l1/readme.md @@ -1,7 +1,7 @@ TEE Searcher === -Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect and Titan Builder's bottom of block. +Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun transactions with full information, without exposing frontrunning risks. This product is currently live on Ethereum mainnet for searching on Flashbots Protect and BuilderNet's bottom of block. - [TDX Mental Model](#tdx-mental-model) - [Image Overview](#image-overview) @@ -10,7 +10,7 @@ Using Intel TDX, Flashbots has built a way for searchers to trustlessly backrun - [Attestation Walkthrough](#attestation-walkthrough) - [Order Flow APIs](#order-flow-apis) - [Flashbots Protect](#searching-on-flashbots-protect-transactions) - - [Titan Builder](#searching-on-titan-builders-bottom-of-block) + - [BuilderNet](#searching-on-buildernets-bottom-of-block) - [Disk Persistence](#disk-persistence) - [Searcher Commands and Services](#searcher-commands-and-services) - [Developer Notes](#developer-notes) @@ -72,13 +72,12 @@ Firewall Rules | 30303 | Input + Output | Execution Client P2P | Podman | TCP + UDP | DISABLED | ENABLED | | 9000 | Input + Output | Consensus Client P2P | Podman | TCP + UDP | ENABLED | ENABLED | | 443 | Output **IP WHITELISTED** | Flashbots Protect Tx Stream | Podman | TCP | ENABLED | DISABLED | -| 42203 | Output **IP WHITELISTED** | Titan Builder State Diff Stream | Podman | TCP | ENABLED | DISABLED | +| 443 | Output **IP WHITELISTED** | BuilderNet State Diff Stream + Bundle RPC | BuilderNet RPC | TCP | ENABLED | DISABLED | | 443 | Output **IP WHITELISTED** | Flashbots Bundle RPC | Flashbots Bundle RPC | TCP | ENABLED | ENABLED | -| 1338 | Output **IP WHITELISTED** | Titan Bundle RPC | Titan Bundle RPC | TCP | ENABLED | ENABLED | -| 54 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | +| 53 | Output | DNS | DNS | TCP + UDP | DISABLED | ENABLED | | 80 | Output | HTTP | HTTP | TCP | DISABLED | ENABLED | | 443 | Output | HTTPS | HTTPS | TCP | DISABLED | ENABLED | -| 8745 | Input | CVM-Reverse-Proxy | Host | TCP | ENABLED | ENABLED | +| 8745 | Input | attested-tls-proxy | Host | TCP | ENABLED | ENABLED | | 123 | Output | NTP | Host | UDP | ENABLED | ENABLED | **Searcher Network Namespace iptables** @@ -100,34 +99,33 @@ iptables only covers ipv4. For security purposes, we block ipv6 with a kernel fl Machine Specs and Cost ------------------------ -Currently, we deploy Azure’s [DCesv5-series Confidential VMs](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/general-purpose/dcesv5-series?tabs=sizebasic). Unfortunately, these are expensive. For reference, Flashbots production TDX builders run in [Standard_EC32es_v5](https://buildernet.org/docs/operating-a-node#microsoft-azure-cloud) with 32 vCPUs and 2TB Disk, which is $2600/month. [Egress](https://azure.microsoft.com/en-us/pricing/details/bandwidth/) (data transferred out of Azure data centers) costs ~$0.087/GB, and historically this costs TEE searchers $150/month. +We deploy GCP [Confidential VMs](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview) on the **c3-standard** machine series (Intel Sapphire Rapids) with Intel TDX support, in **`us-east4` (Northern Virginia)** to colocate with builders. Lighthouse (consensus) runs on the host, while the searcher runs their own execution client (modified Geth or Reth) and bot inside the container. In the future, we hope to add bare metal support, which will lower this cost dramatically. -We place searcher machines in Azure US East 2 to colocate with builders. - -**Machine** -| Name | CPU | Mem (GB) | Price (USD) | -|------------|-----|----------|-------------| -| DC2es_v5 | 2 | 8 | $70.08 | -| DC4es_v5 | 4 | 16 | $140.16 | -| DC8es_v5 | 8 | 32 | $280.32 | -| DC16es_v5 | 16 | 64 | $560.64 | -| DC32es_v5 | 32 | 128 | $1,121.28 | -| DC48es_v5 | 48 | 192 | $1,681.92 | -| DC64es_v5 | 64 | 256 | $2,242.56 | -| DC96es_v5 | 96 | 384 | $3,363.84 | - -**Disk** -| Size | Price (USD) | -|------|-------------| -| 1TB | $123 | -| 2TB | $235 | -| 4TB | $450 | +**Machine** (base price, VM only) + +`c3-standard-22` is the minimum recommended size. + +| Name | vCPU | Mem (GB) | Price (USD/mo) | +|----------------|------|----------|----------------| +| c3-standard-8 | 8 | 32 | ~$294 | +| c3-standard-22 | 22 | 88 | ~$810 | +| c3-standard-44 | 44 | 176 | ~$1,619 | +| c3-standard-88 | 88 | 352 | ~$3,239 | + +**Disk** (price per TB / month, billed on top of the machine) + +| Type | Price (USD/TB) | Per GB | +|-------------|----------------|-----------| +| pd-balanced | ~$110 | ~$0.11/GB | +| pd-ssd | ~$190 | ~$0.19/GB | + +Total monthly cost is machine + disk — e.g. a `c3-standard-22` with 4 TB pd-balanced ≈ $810 + $440 = **~$1,250/mo**. **Egress** -~$0.087/GB, current TEE searchers pay ~$150/month +Networking/egress and the TDX premium are excluded from the estimates above for now. Attestation Walkthrough ------------------------ @@ -184,7 +182,7 @@ During container startup, OpenSSH is installed and the SSH key is copied from `e **Searcher Disk Encryption** -On the first startup, after the searcher's SSH key is received and stored, the searcher must SSH into the machine and run the `initialize` command to encrypt their disk. +On the first startup, after the searcher's SSH key is received and stored, the searcher must SSH into the machine and run the `initialize` command to encrypt their disk. Before this first SSH, complete the [Attestation Walkthrough](#attestation-walkthrough) and bind the host key served at `/pubkey` — the attested channel is up at boot, so this initial connection is verified rather than trust-on-first-use. `Tdx-init` prompts the searcher for a passphrase via stdin, [formats]((https://github.com/flashbots/tdx-init/blob/c357e1b5d9bc386c3446e87bddb6dd53ac01ea97/passphrase.go#L43)) the disk with LUKS2 encryption using this passphrase, and [embeds]((https://github.com/flashbots/tdx-init/blob/c357e1b5d9bc386c3446e87bddb6dd53ac01ea97/passphrase.go#L77)) the searcher's SSH key as metadata in the LUKS header. @@ -282,32 +280,37 @@ Then, copy and paste PCR 4, 9, and 11 into the following format and save as `mea ### 3. audit and run the remote attestation software which requests the measurement from Azure’s vTPM -Flashbots again leverages Edgeless Constellation’s [attested TLS](https://docs.edgeless.systems/constellation/architecture/attestation#attested-tls-atls) and other attestation primitives to interact with Azure’s attestation service. CVM-reverse-proxy fetches Azure's vTPM measurement and compares it with the locally supplied measurement. +Flashbots again leverages Edgeless Constellation’s [attested TLS](https://docs.edgeless.systems/constellation/architecture/attestation#attested-tls-atls) and other attestation primitives to interact with Azure’s attestation service. attested-tls-proxy fetches Azure's vTPM measurement and compares it with the locally supplied measurement. ```bash # download remote attestation tool -git clone https://github.com/flashbots/cvm-reverse-proxy.git -cd cvm-reverse-proxy -make build-proxy-client +git clone https://github.com/flashbots/attested-tls-proxy.git +cd attested-tls-proxy # This will run the client proxy that is listening on port 8080 # and use the server reverse proxy on the deployed image as a target, # marshalling the measurements.json for validation of the attestation. -./cvm-reverse-proxy/build/proxy-client \ ---server-measurements ./measurements.json \ ---target-addr=https://:8745 \ ---log-debug=false +cargo run -- client \ + --listen-addr 127.0.0.1:8080 \ + --allow-self-signed \ + --measurements-file ./measurements.json \ + --log-debug \ + :8745 # To trigger remote attestation, open a new terminal and run this command: curl http://127.0.0.1:8080 -# Bind the expected openssh server pubkey to the attested machine IP -# This command ensures that the ssh server the searcher is connecting to -# is indeed the ssh server that is running on the attested machine. +# Bind the attested host keys to known_hosts. +# This ensures the ssh server the searcher connects to is the one running on the +# attested machine. The attested :8745 channel and the host (dropbear) control-plane +# key are available at boot — before you run `initialize` — so the very first SSH +# is verified rather than trust-on-first-use. /pubkey returns whatever host keys +# are currently available; the container (data-plane) key joins once the disk is +# unlocked and the container is up, so re-run this then to bind it too. git clone https://github.com/flashbots/ssh-pubkey-server ./ssh-pubkey-server/cmd/cli/add_to_known_hosts.sh \ -./cvm-reverse-proxy/build/proxy-client \ +http://127.0.0.1:8080 \ ``` @@ -315,24 +318,35 @@ git clone https://github.com/flashbots/ssh-pubkey-server Example Output ```bash - # successful attestation - ubuntu@schmangeLina-bob-mkosi-builder:~$ ./cvm-reverse-proxy/build/proxy-client \ - --server-measurements ./measurements.json \ - --target-addr=https://20.57.71.148:8745 \ - --log-debug=false - time=2025-07-23T14:00:33.436Z level=INFO msg="Starting proxy client" service=proxy-client version=v0.1.7-1-g4e175a4 listenAddr=127.0.0.1:8080 - time=2025-07-23T14:00:41.224Z level=INFO msg="Validating attestation document" service=proxy-client version=v0.1.7-1-g4e175a4 - time=2025-07-23T14:00:41.956Z level=INFO msg="Successfully validated attestation document" service=proxy-client version=v0.1.7-1-g4e175a4 - time=2025-07-23T14:00:42.051Z level=INFO msg="[proxy-request] proxying complete" service=proxy-client version=v0.1.7-1-g4e175a4 duration=1.275184102s - - # fetch openssh server pubkey - ubuntu@schmangeLina-bob-mkosi-builder::~$ curl --insecure https://20.57.71.148:8745/pubkey - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIYZkgqUokLPpIENJPhJAdpNTecgp/1R1RE6XMsIp6Rt - + # Start the proxy client + $ cargo run -- client --listen-addr 127.0.0.1:8080 --allow-self-signed --measurements-file ./measurements.json --log-debug 35.255.95.67:8745 + Compiling attested-tls-proxy v1.1.1 (/home/pumkin/src/flashbots/attested-tls-proxy) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.17s + Running `target/debug/attested-tls-proxy client --listen-addr '127.0.0.1:8080' --allow-self-signed --measurements-file ./measurements.json --log-debug '35.255.95.67:8745'` + 2026-06-22T07:07:38.553942Z DEBUG attested_tls_proxy: [proxy-client] Connected to proxy server with measurements: Some(DCAP({MRTD: "feb7486608382c1ff0e15b4648ddc0acea6ca974eb53e3529f4c4bd5ffbaa20bf335cb75965cea65fe473aed9647c162", RTMR0: "e1d0235496f93f9475bf0b26d33da5c15831cfc94104d6bea7ab82db027c5f1e917d47dda6953eefae7dcb20ab6f75c4", RTMR1: "4ea5a990afef023f89e11fc32d99103d0adc91d5734664542eb980cdabc88224e1fd206d1d3b2eda71f713fdf8308a2b", RTMR2: "c42ba4fb83f99e4b90bc2a3aa9a2e81c5ac578e9a439c23b457ff7dd0d0eae958760616eff05d827289e47608e757547", RTMR3: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"})) + at src/lib.rs:674 + + 2026-06-22T07:07:38.554036Z DEBUG attested_tls_proxy::http_version: [client] Negotiated ALPN Some("flashbots-ratls/1+h2"), chosen protocol Http2 + at src/http_version.rs:39 + + 2026-06-22T07:07:44.356260Z DEBUG attested_tls_proxy: proxy-client accepted connection + at src/lib.rs:594 + + 2026-06-22T07:07:44.356578Z DEBUG attested_tls_proxy: [proxy-client] Read incoming request from source client: Request { method: GET, uri: /pubkey, version: HTTP/1.1, headers: {"host": "127.0.0.1:8080", "user-agent": "curl/8.19.0", "accept": "*/*"}, body: Body(Empty) } + at src/lib.rs:494 + + 2026-06-22T07:07:44.639534Z DEBUG attested_tls_proxy: [proxy-client] Read response from proxy-server: Response { status: 200, version: HTTP/2.0, headers: {"date": "Mon, 22 Jun 2026 07:07:44 GMT", "content-length": "80", "content-type": "text/plain; charset=utf-8"}, body: Body(Streaming) } + at src/lib.rs:498 + + $ curl http://127.0.0.1:8080/pubkey | less + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 80 100 80 0 0 280 0 0 + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHqPgQoc6yLlMsRUvEeV+6oUsvCMV1sr+b1uKTLzu4e9 ``` -If cvm-reverse-proxy returns `Successfully validated attestation document`, the searcher has now verified that they SSH into a genuine TDX VM, running the exact same image as the one they audited locally. In doing so, the searcher has also verified that no one else has access to the container or host, and they can safely upload your arbitrage bot inside ✨🚀 +If attested-tls-proxy client is able to successfully make a connection to the proxy server, the searcher has now verified that they SSH into a genuine TDX VM, running the exact same image as the one they audited locally. In doing so, the searcher has also verified that no one else has access to the container or host, and they can safely upload your arbitrage bot inside ✨🚀 Order Flow APIs ------------------------ @@ -357,18 +371,20 @@ To submit bundles, connect to the server: https://backruns.tee-searcher.flashbots.net ``` -### Searching on Titan Builder's Bottom of Block +### Searching on BuilderNet's Bottom of Block + +BuilderNet serves both the state diff stream and bundle submission over the same HTTPS endpoint (`rpc.buildernet.org`). Because they share one endpoint, both are reachable only in **production mode** — unlike the Flashbots bundle RPC, which is always on. -**Subscribing to Titan's State Diff Stream** +**Subscribing to BuilderNet's State Diff Stream** **Connecting** Connect to the server located at: ``` -wss://fbtee.titanbuilder.xyz:42203 +wss://rpc.buildernet.org/bob ``` -Use the `eth_subscribe` method to subscribe to state diffs: +Use the `eth_subscribe` method to subscribe to state diffs: ```json {"method":"eth_subscribe","params":["flashbots_stateDiffs"]} @@ -461,11 +477,11 @@ You'll start receiving state diffs: -**Sending Bottom of Block Bundles to Titan RPC** +**Sending Bottom of Block Bundles to BuilderNet RPC** Connect to the server located at: ``` -https://fbtee.titanbuilder.xyz:1338 +https://rpc.buildernet.org ``` Use the `eth_sendBobBundle` method to submit bundles: @@ -486,9 +502,7 @@ Use the `eth_sendBobBundle` method to submit bundles: } ``` -Note on `targetPools`: -- Titan Builder will use `targetPools` to determine what other blocks to consider adding the bottom of block bundle to. -- Searchers should include the address of the contract that’s state change causes the arbitrage. For example, the Uni V2 pool address. +Note: `targetUuid` and `targetPools` are currently accepted but ignored by BuilderNet. When support for them is added, their placement in the request may differ from what is shown above. Disk Persistence ------------------------ @@ -608,8 +622,8 @@ Developer Notes 6. Write new text in `bob.log` to the log socket (**name:** searcher-log-writer.service) (**after:** searcher-log-reader.service) 7. Lighthouse (**name:** `lighthouse.service`) (**after:** `/persistent` is mounted) 8. Start the podman container (**name:** `searcher-container.service`) (**after:** `dropbear.service`, `lighthouse.service`, `searcher-firewall.service`, `/persistent` is mounted) -9. SSH pubkey server (**name:** `ssh-pubkey-server.service`) (**after:** `searcher-container.service`) -10. CVM reverse proxy for SSH pubkey server (**name:** `cvm-reverse-proxy.service`) (**after:** `ssh-pubkey-server.service`) +9. SSH pubkey server (**name:** `ssh-pubkey-server.service`) (**after:** `dropbear.service`) — starts at boot and no longer waits for `searcher-container.service`, so `/pubkey` serves the host (dropbear) key before disk init. The container key is served by `/pubkey` lazily once the container writes it. +10. Attested TLS proxy for SSH pubkey server (**name:** `attested-tls-proxy.service`) (**after:** `ssh-pubkey-server.service`) — consequently the attested `:8745` channel is also available at boot, before the searcher's first SSH. ### Testing diff --git a/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config b/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config index bee194c2..95bc5f53 100644 --- a/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config +++ b/modules/flashbox/flashbox-l2/mkosi.extra/etc/bob/firewall-config @@ -5,7 +5,7 @@ SSH_CONTROL_PORT=22 SSH_DATA_PORT=10022 SSH_REGISTER_PORT=8080 -CVM_REVERSE_PROXY_PORT=8745 +ATTESTED_TLS_PROXY_PORT=8745 SEARCHER_INPUT_UDP_PORT=27017 SEARCHER_INPUT_TCP_PORT=27018 @@ -36,10 +36,10 @@ accept_dst_port $CHAIN_ALWAYS_IN tcp $SEARCHER_INPUT_TCP_PORT "Searcher TCP inpu # We assume here that static peers in config are only syn nodes accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$CONFIG_EL_PEERS_IPS" $ENGINE_API_PORT "Engine API" -# CVM reverse-proxy serves server attestation +# attested-tls-proxy serves server attestation # Also forwards request to ssh pubkey server on localhost:5001, # which serves searcher-container openssh server pubkey -accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy" +accept_dst_port $CHAIN_ALWAYS_IN tcp $ATTESTED_TLS_PROXY_PORT "attested-tls-proxy" ########################################################################### # (2) ALWAYS_OUT: Outbound rules that are always applied diff --git a/modules/flashbox/observability/mkosi.conf b/modules/flashbox/observability/mkosi.conf new file mode 100644 index 00000000..f45d7c01 --- /dev/null +++ b/modules/flashbox/observability/mkosi.conf @@ -0,0 +1,8 @@ +[Content] +ExtraTrees=modules/flashbox/observability/mkosi.extra +PostInstallationScripts=modules/flashbox/observability/mkosi.postinst + +Packages=prometheus + prometheus-node-exporter + prometheus-process-exporter + gettext-base diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml b/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml new file mode 100644 index 00000000..033f901d --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml @@ -0,0 +1,5 @@ +process_names: + # Monitor the searcher container (conmon + all children via --children flag) + - name: "searcher-container" + cmdline: + - 'conmon.*searcher-container' diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl b/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl new file mode 100644 index 00000000..c570fec0 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl @@ -0,0 +1,45 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + # Stamped onto every series sent via remote_write so the upstream can + # distinguish samples coming from different flashbox VMs. + external_labels: + host: ${FLASHBOX_VM} + +# Recording rules for aggregated metrics +rule_files: + - /etc/prometheus/recording_rules.yml + +scrape_configs: + # Node exporter on localhost + - job_name: 'node' + static_configs: + - targets: ['localhost:9100'] + metric_relabel_configs: + # Only keep aggregated metrics for remote write + - source_labels: [__name__] + regex: 'node_(cpu|memory|disk|filesystem|network|vmstat)_.*' + action: keep + + # Process exporter for container monitoring + - job_name: 'process' + static_configs: + - targets: ['localhost:9256'] + +# Remote write via the internal Prometheus metrics proxy at the fixed proxy IP, +# so no DNS resolution happens on the VM; TLS SNI + cert validation are +# overridden via tls_config.server_name to match the cert the proxy's upstream +# presents. +remote_write: + - url: https://${PROMETHEUS_PROXY_IP}/workspaces/${METRICS_FLASHBOTS_WORKSPACE}/api/v1/remote_write + write_relabel_configs: + # Only send flashbox: prefixed metrics + - source_labels: [__name__] + regex: 'flashbox:.*' + action: keep + tls_config: + server_name: aps-workspaces.${METRICS_FLASHBOTS_REGION}.amazonaws.com + sigv4: + region: ${METRICS_FLASHBOTS_REGION} + access_key: ${METRICS_FLASHBOTS_ACCESS_KEY} + secret_key: ${METRICS_FLASHBOTS_SECRET_KEY} diff --git a/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml b/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml new file mode 100644 index 00000000..91475c7c --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml @@ -0,0 +1,45 @@ +groups: + # Base metrics — local: prefix means they stay inside the TEE + # (remote_write only forwards flashbox:*) + - name: local_container_metrics + interval: 30s + rules: + # Percent of total machine CPU capacity (normalized by core count, so the + # 0-100 scale holds regardless of how many vCPUs the VM has). + - record: local:container_cpu_percent + expr: > + sum(rate(namedprocess_namegroup_cpu_seconds_total{groupname="searcher-container"}[5m])) * 100 + / count(node_cpu_seconds_total{mode="idle"}) + + # Forwarded metrics — flashbox: prefix, picked up by remote_write + - name: flashbox_health + interval: 30s + rules: + # 1 if at least one process is running in the searcher-container group, + # absent if process-exporter isn't reporting it. + - record: flashbox:container_alive + expr: namedprocess_namegroup_num_procs{groupname="searcher-container"} > bool 0 + + # Spike-guarded: current 15m avg must be under 80%, + # AND the 10m max ending 5m ago must have been under 70% + - record: flashbox:container_average_cpu_is_under_80_percent + expr: > + (avg_over_time(local:container_cpu_percent[15m]) < bool 80) + * (max_over_time(local:container_cpu_percent[10m] offset 5m) < bool 70) + + - record: flashbox:container_oom_kills_count + expr: node_vmstat_oom_kill + + - record: flashbox:disk_free_space_is_over_10_percent + expr: > + (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) > bool 0.1 + + - record: flashbox:disk_free_space_is_over_128_gb + expr: > + (node_filesystem_avail_bytes{mountpoint="/persistent"}) > bool (128 * 1024 * 1024 * 1024) + + - record: flashbox:network_is_up + expr: > + (sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m])) + + sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))) + > bool 0 diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service new file mode 100644 index 00000000..a8fcf4f7 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service @@ -0,0 +1,14 @@ +[Unit] +Description=Flashbox observability setup (fetch creds, render Prometheus config) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/flashbox-observability-setup +RemainAfterExit=yes +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service new file mode 100644 index 00000000..d21a36bc --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/node-exporter.service @@ -0,0 +1,26 @@ +[Unit] +Description=Prometheus Node Exporter +Documentation=https://github.com/prometheus/node_exporter +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus-node-exporter \ + --web.listen-address=127.0.0.1:9100 \ + --collector.disable-defaults \ + --collector.cpu \ + --collector.meminfo \ + --collector.diskstats \ + --collector.filesystem \ + --collector.netdev \ + --collector.loadavg \ + --collector.vmstat \ + --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker)($|/) +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service new file mode 100644 index 00000000..30b1257c --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/process-exporter.service @@ -0,0 +1,19 @@ +[Unit] +Description=Prometheus Process Exporter +Documentation=https://github.com/ncabatoff/process-exporter +After=network-online.target searcher-container.service +Wants=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus-process-exporter \ + --web.listen-address=127.0.0.1:9256 \ + --config.path=/etc/prometheus/process-exporter.yml \ + --children +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service new file mode 100644 index 00000000..17eafe39 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/etc/systemd/system/prometheus.service @@ -0,0 +1,29 @@ +[Unit] +Description=Prometheus Monitoring System +Documentation=https://prometheus.io/docs/introduction/overview/ +After=network-online.target flashbox-observability-setup.service +Wants=network-online.target + +# flashbox-observability-setup writes this file only when Vault creds are +# fetched + validated. If it's absent (Vault down, dev image, bad secret) the +# unit is cleanly skipped — inactive, not failed, no Restart= crash-loop — and +# self-heals on the next boot that renders a config. +ConditionPathExists=/etc/prometheus/prometheus.yml + +[Service] +Type=simple +User=prometheus +Group=prometheus +ExecStart=/usr/bin/prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/var/lib/prometheus/ \ + --storage.tsdb.retention.time=24h \ + --web.console.templates=/usr/share/prometheus/consoles \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.listen-address=127.0.0.1:9090 +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=minimal.target diff --git a/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup b/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup new file mode 100755 index 00000000..7cf4f37e --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/usr/bin/flashbox-observability-setup @@ -0,0 +1,59 @@ +#!/bin/bash +set -eu -o pipefail + +# Boot-time observability setup: authenticate to Vault, fetch + validate the +# Prometheus remote_write secret, and render /etc/prometheus/prometheus.yml. +# +# Prometheus ships flashbox:* metrics to Prometheus via the internal TCP-proxy +# at a fixed IP (10.88.0.100); the host firewall statically allows that IP, and +# TLS is validated via tls_config.server_name in the template. +# +# Fail-safe: on any failure (QEMU dev, Vault unreachable, missing/invalid +# secret) this writes NO config and exits 0. Because prometheus.service has +# ConditionPathExists=/etc/prometheus/prometheus.yml, it is cleanly skipped +# (inactive, not failed) until a later boot renders a config — boot never +# fails because of observability. + +PROMETHEUS_CONFIG=/etc/prometheus/prometheus.yml +PROMETHEUS_TEMPLATE=/etc/prometheus/prometheus.yml.tmpl + +# Internal metrics proxy IP, injected into the template so the literal isn't +# duplicated there. Must match the host firewall rule (firewall-config) and the +# container netns DROP (init-container.sh). +PROMETHEUS_PROXY_IP=10.88.0.100 +export PROMETHEUS_PROXY_IP + +. /usr/lib/flashbox/vault.sh + +# Always start clean — never leave a stale config from a previous boot. +rm -f "$PROMETHEUS_CONFIG" + +# host= external label, from the GCE instance name. +FLASHBOX_VM=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/name") || FLASHBOX_VM=unknown +echo "$FLASHBOX_VM" | grep -qE '^[a-z0-9-]+$' || FLASHBOX_VM=unknown +export FLASHBOX_VM + +# Vault auth uses the GCE instance-identity JWT, so it only works on GCP. +if ! grep -q Google /sys/class/dmi/id/sys_vendor 2>/dev/null; then + echo "Not running on GCP, leaving Prometheus disabled" + exit 0 +fi + +# Fetch + validate the four metrics variables (exported by vault_fetch). +if ! vault_fetch; then + echo "WARNING: Vault fetch/validation failed, leaving Prometheus disabled" + exit 0 +fi + +# The rendered config holds the SigV4 secret +touch "$PROMETHEUS_CONFIG" +chown root:prometheus "$PROMETHEUS_CONFIG" +chmod 640 "$PROMETHEUS_CONFIG" + +# Single render: only the curated variables are substituted; every other token in +# the template (e.g. [__name__]) is left untouched. +envsubst '$FLASHBOX_VM $PROMETHEUS_PROXY_IP $METRICS_FLASHBOTS_WORKSPACE $METRICS_FLASHBOTS_REGION $METRICS_FLASHBOTS_ACCESS_KEY $METRICS_FLASHBOTS_SECRET_KEY' \ + < "$PROMETHEUS_TEMPLATE" > "$PROMETHEUS_CONFIG" + +echo "Observability config written (workspace: ${METRICS_FLASHBOTS_WORKSPACE})" diff --git a/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh b/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh new file mode 100644 index 00000000..6bca21b6 --- /dev/null +++ b/modules/flashbox/observability/mkosi.extra/usr/lib/flashbox/vault.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Vault GCP auth + fetch of the Prometheus remote_write secret. +# Sourced by flashbox-observability-setup. +# +# vault_fetch logs in with the GCE instance-identity JWT, reads the shared +# secret, and exports the four METRICS_FLASHBOTS_* vars. Each value is +# format-checked before export; any failure returns non-zero and the caller +# writes no config. + +vault_fetch() { + local addr mount role kv suffix jwt token data + + # curl --retry: this runs early in boot, so the metadata server and Vault + # may not be reachable on the first try. Without retries a transient miss + # would disable metrics until the next reboot. + + # 1. Bootstrap config from GCE instance metadata. + addr=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_addr") || return 1 + mount=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_auth_mount_gcp") || return 1 + role=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_role") || return 1 + kv=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_kv_path") || return 1 + suffix=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/vault_kv_common_suffix") || return 1 + + # 2. Authenticate: GCE identity JWT -> Vault token. + jwt=$(curl -sf --retry 5 --retry-connrefused --header "Metadata-Flavor: Google" \ + --data-urlencode "audience=http://vault/${role}" \ + --data-urlencode "format=full" \ + "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity") || return 1 + token=$(curl -sf --retry 5 --retry-connrefused \ + --data "$(printf '{"role":"%s","jwt":"%s"}' "$role" "$jwt")" \ + "${addr}/v1/${mount}/login" | jq -re .auth.client_token) || return 1 + + # 3. Read the shared secret blob. + data=$(curl -sf --retry 5 --retry-connrefused --header "X-Vault-Token: ${token}" \ + "${addr}/v1/${kv}/node/${suffix}" | jq -ce .data.data) || return 1 + + # 4. Extract each variable and validate it against its expected format. + METRICS_FLASHBOTS_WORKSPACE=$(echo "$data" | jq -re .METRICS_FLASHBOTS_WORKSPACE) || return 1 + echo "$METRICS_FLASHBOTS_WORKSPACE" | grep -qE '^ws-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' \ + || { echo "vault_fetch: WORKSPACE is not an AMP workspace id" >&2; return 1; } + + METRICS_FLASHBOTS_REGION=$(echo "$data" | jq -re .METRICS_FLASHBOTS_REGION) || return 1 + echo "$METRICS_FLASHBOTS_REGION" | grep -qE '^[a-z]{2}-[a-z]+-[0-9]+$' \ + || { echo "vault_fetch: REGION is not an AWS region" >&2; return 1; } + + METRICS_FLASHBOTS_ACCESS_KEY=$(echo "$data" | jq -re .METRICS_FLASHBOTS_ACCESS_KEY) || return 1 + echo "$METRICS_FLASHBOTS_ACCESS_KEY" | grep -qE '^[A-Z0-9]{20}$' \ + || { echo "vault_fetch: ACCESS_KEY is not an AWS access key id" >&2; return 1; } + + METRICS_FLASHBOTS_SECRET_KEY=$(echo "$data" | jq -re .METRICS_FLASHBOTS_SECRET_KEY) || return 1 + echo "$METRICS_FLASHBOTS_SECRET_KEY" | grep -qE '^[A-Za-z0-9/+]{40}$' \ + || { echo "vault_fetch: SECRET_KEY is not an AWS secret key" >&2; return 1; } + + # 5. All present and well-formed — publish to the environment for envsubst. + export METRICS_FLASHBOTS_WORKSPACE METRICS_FLASHBOTS_REGION \ + METRICS_FLASHBOTS_ACCESS_KEY METRICS_FLASHBOTS_SECRET_KEY +} diff --git a/modules/flashbox/observability/mkosi.postinst b/modules/flashbox/observability/mkosi.postinst new file mode 100755 index 00000000..c32a35f3 --- /dev/null +++ b/modules/flashbox/observability/mkosi.postinst @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Ensure prometheus owns its data directory +mkosi-chroot chown -R prometheus:prometheus /var/lib/prometheus + +# Enable observability services +mkosi-chroot systemctl add-wants minimal.target \ + flashbox-observability-setup.service \ + prometheus.service \ + node-exporter.service \ + process-exporter.service diff --git a/scripts/gcp_measurements_to_dcap.sh b/scripts/gcp_measurements_to_dcap.sh new file mode 100755 index 00000000..a498a169 --- /dev/null +++ b/scripts/gcp_measurements_to_dcap.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# TEMPORARY: reshape dstack-mr GCP measurements (make measure-gcp -> +# build/gcp_measurements.json) into attested-tls-proxy's dcap-tdx +# --measurements-file format. Drop once attested-tls-proxy accepts the +# condensed dstack-mr output directly. +# +# dstack-mr emits mrtd[] (one per known GCP firmware) and rtmr0[] (equal-size +# per-firmware chunks: firmware x machine-type-ACPI x boot variants), while +# attested-tls-proxy wants flat measurement sets it ORs over. We pair mrtd[i] +# with its rtmr0 chunk; a genuine quote matches exactly one set. +# +# Usage: scripts/gcp_measurements_to_dcap.sh [gcp_measurements.json] > out.json +set -euo pipefail +jq ' + (.mrtd | length) as $m + | (((.rtmr0 | length) / $m) | floor) as $c + | [ range(0; $m) as $i + | range(0; $c) as $j + | { measurement_id: "local fw\($i)-v\($j)", + attestation_type: "dcap-tdx", + measurements: { + "0": { expected: .mrtd[$i] }, + "1": { expected: .rtmr0[$i*$c + $j] }, + "2": { expected: .rtmr1 }, + "3": { expected: .rtmr2 }, + "4": { expected: .rtmr3 } + } } ] +' "${1:-build/gcp_measurements.json}" diff --git a/scripts/make_git_package.sh b/scripts/make_git_package.sh index 2a32a766..f35cb17c 100644 --- a/scripts/make_git_package.sh +++ b/scripts/make_git_package.sh @@ -43,7 +43,7 @@ make_git_package() { mkdir -p "$DESTDIR$dest" cp -r "$cache_dir/$src"/* "$DESTDIR$dest/" else - cp "$cache_dir/$src" "$DESTDIR$dest" + install -m 755 "$cache_dir/$src" "$DESTDIR$dest" fi done return 0 @@ -66,7 +66,7 @@ make_git_package() { cp -r "$build_dir/$src"/* "$DESTDIR$dest/" else mkdir -p "$(dirname "$DESTDIR$dest")" - cp "$build_dir/$src" "$DESTDIR$dest" + install -m 755 "$build_dir/$src" "$DESTDIR$dest" fi # Cache artifact