feat(daemon): adopt mimalloc-pprof and prove heap, on-CPU, and off-CPU profiling

Cargo.toml

@@ -127,9 +127,22 @@ async-trait = "0.1"
 dashmap = "6"
 blake3 = "1"
 criterion = { version = "0.5", default-features = false, features = ["html_reports"] }
-mimalloc = "0.1"
+# Global allocator for `fbuild-daemon`. `mimalloc-pprof` vendors the same
+# mimalloc (v3 line) plus a sampled heap profiler that stays dormant until
+# started, so this is the plain allocator it replaced with a reporting half
+# attached — see FastLED/fbuild#1361 and the ~3.9 GB idle daemon in #1360.
+# Not feature-gated on purpose: a profiler compiled out of the shipped
+# binary is never present on the machine where a slow leak reproduces.
+mimalloc-pprof = "0.9"
+
+# On-CPU sampling, stack symbolization, and the off-CPU/async profile
+# pipeline. Dev-only: this is how the profiling contract is tested, not
+# something the shipped daemon links. The git rev MUST equal the
+# `running-process` rev pinned above, for the same `rp_*_public`
+# symbol-duplication reason documented there.
+running-process-probe-daemon = { version = "4.8.1", git = "https://github.com/zackees/running-process.git", rev = "359b4e3d92660e54eb9b16b6e83de8ce26932ff8" }
 object = { version = "0.36", default-features = false, features = ["read", "std", "elf", "write"] }
-rusqlite = { version = "0.31", features = ["bundled"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
 shell-words = "1"
 bincode = "1"
 # USB identity data is fetched from FastLED/boards at runtime. Do not add a

crates/fbuild-core/Cargo.toml

@@ -83,3 +83,15 @@ widestring = { workspace = true }
 tempfile = { workspace = true }
 # Test-only embedded USB-vendor fixture extraction. Production never links it.
 tar = { workspace = true }
+# FastLED/fbuild#1361: the on-CPU sampler, its frame symbolizer, and the
+# off-CPU/async profile pipeline, exercised by tests/cpu_profiling.rs.
+#
+# It lives here rather than on fbuild-daemon for a linker reason, not a
+# taste one. `running-process-probe` pulls crash-handler 0.7 while the
+# pinned zccache 1.13.1 pulls 0.6.3, and both export the same unmangled C
+# symbols (`ehsetjmp`, `handle_invalid_parameter`, ...). Any binary
+# linking both fails with duplicate symbols. fbuild-core is the deepest
+# crate that does NOT depend on zccache, so its test binary links exactly
+# one copy. The heap half of #1361 stays in fbuild-daemon, which needs no
+# probe crate at all.
+running-process-probe-daemon = { workspace = true }

crates/fbuild-core/tests/cpu_profiling.rs

@@ -0,0 +1,189 @@
+//! On-CPU and off-CPU profiling, exercised end to end (FastLED/fbuild#1361).
+//!
+//! Two different questions, and fbuild needs both answered:
+//!
+//! - **on-CPU** — "what is running?" Sampled stacks, then symbolized off the
+//!   hot path.
+//! - **off-CPU** — "what is *waiting*?" This is the mode that matters most for
+//!   `fbuild-daemon`, which spends nearly all its wall clock blocked on
+//!   subprocesses, sockets, and the filesystem. A CPU profile is blind to all
+//!   of that: a request that spent nine seconds waiting and one computing
+//!   looks, in a CPU profile, like one second of work.
+//!
+//! These live in `fbuild-core` rather than next to the heap test in
+//! `fbuild-daemon` for a linker reason, documented on the dev-dependency in
+//! this crate's `Cargo.toml`: `running-process-probe` and the pinned zccache
+//! disagree about `crash-handler`, and both export the same unmangled C
+//! symbols, so no single binary can link both.
+
+use std::time::Duration;
+
+// ---------------------------------------------------------------------------
+// On-CPU — sampled stacks plus symbolization
+// ---------------------------------------------------------------------------
+
+/// The frame the on-CPU profile has to find.
+///
+/// `#[inline(never)]` because the point is that this symbol survives into the
+/// sampled stack; inlined into its caller it would have no address of its own
+/// for samples to land on.
+#[inline(never)]
+fn fbuild_on_cpu_hot_loop(stop: &std::sync::atomic::AtomicBool) -> u64 {
+    let mut acc = 0u64;
+    while !stop.load(std::sync::atomic::Ordering::Relaxed) {
+        for i in 0..4096u64 {
+            acc = acc.wrapping_add(i).wrapping_mul(31);
+        }
+        std::hint::black_box(acc);
+    }
+    acc
+}
+
+#[test]
+fn on_cpu_profiling_samples_a_busy_thread_and_attributes_the_frames() {
+    use running_process_probe_daemon::profile::session::{ProfileRequest, ProfileSession};
+    use running_process_probe_daemon::profile::symbolize::ModuleResolver;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicBool, Ordering};
+
+    // The sampler suspends *sibling* threads, so the work being profiled must
+    // not sit on the thread calling `run()` — that thread is the profiler.
+    let stop = Arc::new(AtomicBool::new(false));
+    let worker_stop = Arc::clone(&stop);
+    let worker = std::thread::spawn(move || fbuild_on_cpu_hot_loop(&worker_stop));
+
+    let session = ProfileSession::new(ProfileRequest {
+        hz: 99,
+        duration: Duration::from_millis(400),
+    });
+    let metrics = session.run();
+
+    stop.store(true, Ordering::Relaxed);
+    let acc = worker.join().expect("worker thread must not panic");
+    std::hint::black_box(acc);
+
+    assert!(
+        metrics.samples_captured > 0,
+        "a 400 ms session at 99 Hz over a busy thread must capture samples; \
+         captured={} dropped={} threads_seen={}",
+        metrics.samples_captured,
+        metrics.samples_dropped,
+        metrics.threads_seen
+    );
+    assert!(
+        metrics.threads_seen >= 1,
+        "at least the busy worker thread must appear in the profile"
+    );
+
+    // Symbolization. `ModuleResolver` is deliberately the floor of what can be
+    // resolved without symbol files: it attributes each address to its owning
+    // module plus an ASLR-independent offset, and stays honest about the rest
+    // by naming unresolved frames `module+0xoffset` rather than inventing a
+    // function name. Asserting DWARF/PDB function names here would be
+    // asserting on the build's debug-info settings rather than on the
+    // profiler; #1361 tracks that separately as the release-profile question.
+    let mut resolver = ModuleResolver::for_current_process()
+        .expect("module enumeration must work on a supported host");
+    assert!(
+        resolver.module_count() > 0,
+        "the current process must have at least one loaded module"
+    );
+    let resolved = session.resolve(&mut resolver, metrics);
+
+    let folded = resolved.folded();
+    assert!(
+        !folded.is_empty(),
+        "resolved samples must fold into at least one stack"
+    );
+    let attributed = folded
+        .iter()
+        .flat_map(|(stack, _)| stack.iter())
+        .any(|frame| !frame.is_empty());
+    assert!(
+        attributed,
+        "every folded stack was empty — frames reached no module at all"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Off-CPU — the async/waiting pipeline
+// ---------------------------------------------------------------------------
+
+#[test]
+fn off_cpu_profiling_ranks_waiting_above_running() {
+    use running_process_probe_daemon::profile::async_profile::{
+        CustomAdapter, TaskSample, profile, to_collapsed, to_pprof,
+    };
+
+    // Two tasks with inverted profiles: one waits nine seconds and works for
+    // one, the other works constantly. In an on-CPU profile the busy task
+    // dominates and the waiter is invisible. An off-CPU profile has to invert
+    // that — surfacing the waiter is the entire reason to take one.
+    let waiting = TaskSample {
+        spawn_stack: vec![
+            "fbuild_daemon::main".to_string(),
+            "fbuild_daemon::handlers::operations::build".to_string(),
+            "fbuild_build::compile_many::await_subprocess".to_string(),
+        ],
+        idle_nanos: 9_000_000_000,
+        busy_nanos: 1_000_000_000,
+        scheduled_nanos: 12_000,
+        polls: 3,
+        wakes: 3,
+        name: "compile-wait".to_string(),
+    };
+    let busy = TaskSample {
+        spawn_stack: vec![
+            "fbuild_daemon::main".to_string(),
+            "fbuild_daemon::status_manager::tick".to_string(),
+        ],
+        idle_nanos: 1_000_000,
+        busy_nanos: 500_000_000,
+        scheduled_nanos: 4_000,
+        polls: 900,
+        wakes: 900,
+        name: "status-tick".to_string(),
+    };
+
+    let collected = {
+        let samples = vec![waiting.clone(), busy.clone()];
+        let mut adapter = CustomAdapter::new(move |_window| samples.clone());
+        profile(&mut adapter, Duration::from_secs(5)).expect("adapter must produce samples")
+    };
+    assert_eq!(collected.len(), 2, "both tasks must survive collection");
+
+    let pprof = to_pprof(&collected);
+    assert!(
+        !pprof.is_empty(),
+        "off-CPU samples must serialize to a pprof profile"
+    );
+
+    let collapsed = to_collapsed(&collected);
+    assert!(
+        collapsed.contains("await_subprocess"),
+        "the waiting task's spawn stack must appear in the off-CPU profile:\n{collapsed}"
+    );
+
+    // The ranking, not just the presence. Collapsed output opens on idle time,
+    // so the waiter has to weigh more than the busy task even though the busy
+    // task used 500x more CPU.
+    let waiting_weight = collapsed_weight(&collapsed, "await_subprocess");
+    let busy_weight = collapsed_weight(&collapsed, "status_manager::tick");
+    assert!(
+        waiting_weight > busy_weight,
+        "off-CPU profile must rank waiting above running \
+         (waiting={waiting_weight}, busy={busy_weight}):\n{collapsed}"
+    );
+}
+
+/// Sum the counts of collapsed lines whose stack mentions `needle`.
+///
+/// Collapsed format is `frame;frame;frame <count>` per line.
+fn collapsed_weight(collapsed: &str, needle: &str) -> u64 {
+    collapsed
+        .lines()
+        .filter(|line| line.contains(needle))
+        .filter_map(|line| line.rsplit_once(' '))
+        .filter_map(|(_stack, count)| count.trim().parse::<u64>().ok())
+        .sum()
+}

crates/fbuild-daemon/Cargo.toml

@@ -60,7 +60,7 @@ regex = { workspace = true }
 async-trait = { workspace = true }
 tempfile = { workspace = true }
 
-[dependencies.mimalloc]
+[dependencies.mimalloc-pprof]
 workspace = true
 
 [dev-dependencies]

crates/fbuild-daemon/src/handlers/health.rs

@@ -2,7 +2,8 @@
 
 use crate::context::DaemonContext;
 use crate::models::{
-    DaemonInfoResponse, HealthResponse, RootResponse, ShutdownParams, ShutdownResponse,
+    DaemonInfoResponse, HealthResponse, HeapDumpResponse, RootResponse, ShutdownParams,
+    ShutdownResponse,
 };
 use axum::Json;
 use axum::extract::{ConnectInfo, Query, State};
@@ -67,6 +68,80 @@ pub async fn daemon_info(State(ctx): State<Arc<DaemonContext>>) -> Json<DaemonIn
     })
 }
 
+/// POST /api/daemon/heap-dump
+///
+/// Write a pprof heap snapshot of this daemon and return where it landed.
+///
+/// Deliberately reachable on a daemon that is already misbehaving, which is
+/// the case FastLED/fbuild#1360 ran into: the process had grown to ~3.9 GB and
+/// restarting it to turn on a profiler would have destroyed the very leak
+/// under investigation.
+///
+/// Starts a session on demand when none is running, so an operator who did
+/// not set `FBUILD_HEAP_PROFILE` at startup still gets something. That
+/// snapshot only covers allocations made *after* this call — the response
+/// says so rather than letting a thin profile read as "nothing is leaking".
+pub async fn heap_dump(
+    ConnectInfo(peer): ConnectInfo<SocketAddr>,
+) -> (StatusCode, Json<HeapDumpResponse>) {
+    // Loopback only. The daemon binds 0.0.0.0 (see `main.rs`), and this
+    // endpoint is not a read: it can switch process-wide profiling on and make
+    // the daemon serialize its whole heap on demand. Reachable from off-box
+    // that is both a denial-of-service lever and a way to extract allocation
+    // shapes from someone else's machine. Nothing about a profiling dump needs
+    // to cross a network boundary, so the check is a flat refusal rather than
+    // a rate limit.
+    if !peer.ip().is_loopback() {
+        tracing::warn!(peer = %peer, "heap-dump refused: non-loopback caller");
+        return (
+            StatusCode::FORBIDDEN,
+            Json(HeapDumpResponse {
+                path: None,
+                live_samples: 0,
+                profiling_was_already_running: crate::heap_profile::is_enabled(),
+                message: "heap-dump is loopback-only".to_string(),
+            }),
+        );
+    }
+
+    let was_running = crate::heap_profile::is_enabled();
+    if !was_running {
+        crate::heap_profile::start(DEFAULT_ON_DEMAND_SAMPLE_RATE);
+    }
+
+    match crate::heap_profile::dump(None).await {
+        Ok(path) => (
+            StatusCode::OK,
+            Json(HeapDumpResponse {
+                path: Some(path.display_slash()),
+                live_samples: crate::heap_profile::live_sample_count(),
+                profiling_was_already_running: was_running,
+                message: if was_running {
+                    "heap snapshot written".to_string()
+                } else {
+                    "heap snapshot written, but profiling only started with this                      request — it covers allocations from now on, not the ones                      already held. Set FBUILD_HEAP_PROFILE=1 before starting the                      daemon to capture from process start."
+                        .to_string()
+                },
+            }),
+        ),
+        Err(error) => (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(HeapDumpResponse {
+                path: None,
+                live_samples: 0,
+                profiling_was_already_running: was_running,
+                message: format!("heap dump failed: {error}"),
+            }),
+        ),
+    }
+}
+
+/// Sample rate used when a dump is requested on a daemon that was not started
+/// with profiling on. 64 KiB is finer than the 512 KiB default: by this point
+/// someone is actively chasing something, and the extra resolution is worth
+/// more than the overhead.
+const DEFAULT_ON_DEMAND_SAMPLE_RATE: usize = 64 * 1024;
+
 /// POST /api/daemon/shutdown
 pub async fn shutdown(
     State(ctx): State<Arc<DaemonContext>>,