From 3a99eb1d954aee5b7610beb7b3deedf1260dc856 Mon Sep 17 00:00:00 2001 From: Aleksander Arruda Date: Thu, 10 Sep 2026 15:33:35 -0300 Subject: [PATCH] Build a native arm64 image instead of emulating amd64 .cargo/config.toml pins -C target-cpu=neoverse-512tvb for aarch64-unknown-linux-gnu, so an arm64 image built from source SIGILLs on anything short of Graviton3+. DOCKER.md worked around it by telling Apple Silicon users to build linux/amd64 and emulate. Assemble RUSTFLAGS in the Dockerfile instead. Overriding the pin needs RUSTFLAGS specifically, not CARGO_TARGET__RUSTFLAGS: the latter is another source for the same config key and cargo joins config arrays, so -C target-cpu would survive. ARM64_TARGET_CPU opts back into the tuning for a Graviton deployment. RUSTFLAGS does not reach a C compiler, and lore-base pins the same -mcpu for its cc build of rpmalloc. That flag raises the architecture floor as well as the tuning -- GCC 14 emits stlur (FEAT_LRCPC2, armv8.4) and ldapr (armv8.3), undefined on Neoverse N1 parts such as Ampere Altra and Graviton2. build.rs now takes LORE_ARM64_TARGET_CPU, defaulting to the value it hardcoded, and the Dockerfile passes the choice through. Verified by disassembling both builds: with the tuning off, the allocator is clean armv8-a. EXPOSE defaults to TCP, so it was not declaring the QUIC listener sharing 41337; docker run -P left it unreachable. Signed-off-by: Aleksander Arruda Co-Authored-By: Claude Opus 5 (1M context) --- contrib/aws/README.md | 8 +++++++- docs/how-to/deploy-local-lore-server.md | 6 +++--- lore-base/build.rs | 10 +++++++++- lore-server/DOCKER.md | 20 +++++++++++++++----- lore-server/Dockerfile | 24 +++++++++++++++++++++--- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/contrib/aws/README.md b/contrib/aws/README.md index 9490bbd5e..7b9214854 100644 --- a/contrib/aws/README.md +++ b/contrib/aws/README.md @@ -13,9 +13,15 @@ This example uses **c8gd.8xlarge** Graviton instances (32 vCPU, 64 GB RAM, 1.9 T From the Lore repo root: ```sh -docker buildx build --platform linux/arm64 -f lore-server/Dockerfile -t loreserver:v0.8.7 --load . +docker buildx build --platform linux/arm64 \ + --build-arg ARM64_TARGET_CPU=neoverse-512tvb \ + -f lore-server/Dockerfile -t loreserver:v0.8.7 --load . ``` +> `ARM64_TARGET_CPU` tunes codegen for Graviton3+, matching the `c8gd` instances below. Without it +> the build is baseline `armv8-a`, which runs here but leaves performance on the table. The +> resulting binary uses SVE and will not run on older arm64 hardware. + > If building on an x86 host, [register QEMU](https://docs.docker.com/build/building/multi-platform/#qemu) first: > `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` diff --git a/docs/how-to/deploy-local-lore-server.md b/docs/how-to/deploy-local-lore-server.md index a389639a2..91d5ecdc9 100644 --- a/docs/how-to/deploy-local-lore-server.md +++ b/docs/how-to/deploy-local-lore-server.md @@ -15,7 +15,7 @@ In this guide, you'll deploy local Lore Servers — with durable storage and a c The binary and Docker paths are mutually exclusive, and each is complete on its own — follow one top to bottom. - **[Run from the binary](#run-from-the-binary):** Fewer moving parts and native performance. Pick this to run `loreserver` directly on the host. -- **[Run with Docker](#run-with-docker):** An isolated container. Pick this if you'd rather not put a binary on the host — but note the `linux/amd64` emulation caveat on Apple Silicon in the build step. +- **[Run with Docker](#run-with-docker):** An isolated container. Pick this if you'd rather not put a binary on the host. Builds natively on both `amd64` and `arm64`, Apple Silicon included. ## Run from the binary @@ -196,11 +196,11 @@ The binary and Docker paths are mutually exclusive, and each is complete on its This needs Docker (and WSL2 on Windows) and the Lore repository cloned locally. Building the image compiles the server, so it needs several GB of free RAM. From the repository root: ```bash - docker build --platform linux/amd64 -f lore-server/Dockerfile -t lore-server . + docker build -f lore-server/Dockerfile -t lore-server . ``` > [!NOTE] - > On Apple Silicon or Windows (both arm64 and amd64), build and run with `--platform linux/amd64` as shown. The `linux/arm64` server image targets AWS Graviton3 (SVE), an instruction set those CPUs lack. + > The image builds for your host architecture. On Apple Silicon or Windows on arm64 that is a baseline `armv8-a` build which runs natively — no `--platform` override is needed. To tune arm64 for AWS Graviton3 and newer instead, add `--build-arg ARM64_TARGET_CPU=neoverse-512tvb`; that binary uses SVE and will not run on other arm64 hardware. 2. **Run it with default settings.** diff --git a/lore-base/build.rs b/lore-base/build.rs index 9ce1f889e..bff501bee 100644 --- a/lore-base/build.rs +++ b/lore-base/build.rs @@ -28,8 +28,16 @@ fn main() -> Result<(), Box> { .define("ENABLE_OVERRIDE", "0") .includes(Some(native_dir.join("thirdparty"))); + // Mirrors the -C target-cpu .cargo/config.toml pins, and settable to empty + // for a baseline build: neoverse-512tvb raises the architecture floor too, + // emitting stlur (armv8.4) that is undefined on Neoverse N1 (Ampere Altra). + println!("cargo:rerun-if-env-changed=LORE_ARM64_TARGET_CPU"); if platform == "linux" && arch == "aarch64" { - cc_builder.flag("-mcpu=neoverse-512tvb"); + let target_cpu = + env::var("LORE_ARM64_TARGET_CPU").unwrap_or_else(|_| "neoverse-512tvb".to_string()); + if !target_cpu.is_empty() { + cc_builder.flag(format!("-mcpu={target_cpu}")); + } } if cc_builder.get_compiler().is_like_msvc() { diff --git a/lore-server/DOCKER.md b/lore-server/DOCKER.md index 688565dd5..4e817c4f0 100644 --- a/lore-server/DOCKER.md +++ b/lore-server/DOCKER.md @@ -6,19 +6,29 @@ telemetry integration, or replication is configured. ## Prerequisites - Docker with BuildKit support -- On Apple Silicon (M-series Macs), builds must target `linux/amd64` due to Graviton-specific - compiler flags in `.cargo/config.toml` for `aarch64-unknown-linux-gnu` + +Both `linux/amd64` and `linux/arm64` build. `.cargo/config.toml` pins `aarch64-unknown-linux-gnu` +to Graviton3+ via `-C target-cpu=neoverse-512tvb`, which faults on older arm64 parts, so the +Dockerfile assembles `RUSTFLAGS` itself and leaves that tuning off by default. An arm64 image you +build here therefore runs on any armv8-a host, Apple Silicon included. ## Building From the repository root: ```sh -docker build --platform linux/amd64 -f lore-server/Dockerfile -t loreserver . +docker build -f lore-server/Dockerfile -t loreserver . ``` -The build compiles the `loreserver` binary and generates self-signed TLS certificates for QUIC -using `scripts/server/make-certs.sh`. +Pass `--platform linux/amd64` or `--platform linux/arm64` to cross-build; expect it to be slow, +since a release Rust build under emulation is far slower than a native one. + +To tune arm64 for Graviton3 and newer, as Lore is deployed, pass the microarchitecture. The +resulting binary will not run on older arm64 hardware: + +```sh +docker build -f lore-server/Dockerfile --build-arg ARM64_TARGET_CPU=neoverse-512tvb -t loreserver . +``` ## Running diff --git a/lore-server/Dockerfile b/lore-server/Dockerfile index 1c3245877..56b19d12c 100644 --- a/lore-server/Dockerfile +++ b/lore-server/Dockerfile @@ -8,10 +8,28 @@ RUN apt-get update && apt-get install -y \ WORKDIR /build COPY . . +# Which arm64 microarchitecture to tune for. Empty is baseline armv8-a, which +# runs anywhere. Ignored on amd64. +ARG TARGETARCH +ARG ARM64_TARGET_CPU="" + +# RUSTFLAGS is restated in full because .cargo/config.toml pins Graviton3+ for +# this target and cargo *joins* config arrays: only RUSTFLAGS replaces [build] +# and [target.*] outright, so anything less leaves a baseline build SIGILLing. +# LORE_ARM64_TARGET_CPU carries the same choice to lore-base's cc build of +# rpmalloc, which RUSTFLAGS cannot reach. RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/target \ - cargo build --release --bin loreserver && \ + set -eu; \ + RUSTFLAGS="--cfg tokio_unstable --cfg uuid_unstable -C force-unwind-tables=yes -C force-frame-pointers=yes"; \ + if [ "${TARGETARCH}" = "arm64" ] && [ -n "${ARM64_TARGET_CPU}" ]; then \ + RUSTFLAGS="${RUSTFLAGS} -C target-cpu=${ARM64_TARGET_CPU}"; \ + fi; \ + export RUSTFLAGS; \ + export LORE_ARM64_TARGET_CPU="${ARM64_TARGET_CPU}"; \ + echo "building with RUSTFLAGS=${RUSTFLAGS} LORE_ARM64_TARGET_CPU=${ARM64_TARGET_CPU}"; \ + cargo build --release --bin loreserver; \ cp /build/target/release/loreserver /build/loreserver-bin FROM debian:trixie-slim @@ -42,7 +60,7 @@ ENV RUST_LOG=info RUN mkdir -p /data VOLUME /data -# QUIC + gRPC: 41337, HTTP: 41339 -EXPOSE 41337 41339 +# 41337 carries gRPC over TCP and QUIC over UDP; both are needed. +EXPOSE 41337/tcp 41337/udp 41339/tcp ENTRYPOINT ["loreserver"]