From 4d72053e770bb8146ceea16f2cdc798bb0765a5e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 03:44:10 +0000 Subject: [PATCH] perf(daemon): stop capping glibc malloc arenas in the service unit The installed systemd unit set MALLOC_ARENA_MAX=2 to bound retained memory. On a 20-worker index the two arenas became the daemon's hot lock: perf on beta.41 put 60% of daemon CPU in __pv_queued_spin_lock_slowpath, every wait under __lll_lock_wait_private from ordinary Vec/String growth, serde_json string decode, and page preparation, while RSS still reached 15 GB. With the cap removed on the same host and corpus the spinlock fell to 4.7% and extraction's share of worker cycles rose from 2% to 23%. The unit leaves the allocator alone; a test keeps it that way. --- crates/tracedecay-cli/src/main.rs | 9 ++++---- .../tracedecay-daemon-control/src/service.rs | 1 - .../src/service/tests.rs | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index afc61d0f1a..d87c593144 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -19,10 +19,11 @@ static HOTPATH_ALLOCATOR: hotpath::CountingAllocator = hotpath::CountingAllocato // may exist per binary, so overlapping selections resolve by fixed precedence // rather than a compile error: hotpath-alloc's counting allocator wins in // measurement builds, then jemalloc, then mimalloc. The default build keeps -// the system allocator (glibc malloc on Linux), whose retained-arena behavior -// the daemon compensates for with `malloc_trim` at maintenance boundaries and -// `MALLOC_ARENA_MAX=2` in the installed service unit; neither compensation is -// load-bearing under jemalloc or mimalloc. +// the system allocator (glibc malloc on Linux). The installed service unit +// must not cap glibc's arenas: `MALLOC_ARENA_MAX=2` once did, to bound +// retained memory, and put 60% of a 20-worker daemon's CPU into two arena +// locks (perf on beta.41: every wait was `__lll_lock_wait_private` under +// ordinary Vec/String growth) while RSS still reached 15 GB. #[cfg(all(feature = "alloc-jemalloc", not(feature = "hotpath-alloc")))] #[global_allocator] static JEMALLOC_ALLOCATOR: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; diff --git a/crates/tracedecay-daemon-control/src/service.rs b/crates/tracedecay-daemon-control/src/service.rs index fd6b113530..97b64877a0 100644 --- a/crates/tracedecay-daemon-control/src/service.rs +++ b/crates/tracedecay-daemon-control/src/service.rs @@ -522,7 +522,6 @@ impl DaemonServiceSpec { [Service]\n\ Type=simple\n\ Environment=\"PATH={}\"\n\ - Environment=\"MALLOC_ARENA_MAX=2\"\n\ ExecStart={} daemon run --socket {}{}\n\ # Restart=always (not on-failure): come back after OOM SIGKILL,\n\ # crash, or a clean-but-unexpected exit. A looping daemon is\n\ diff --git a/crates/tracedecay-daemon-control/src/service/tests.rs b/crates/tracedecay-daemon-control/src/service/tests.rs index b1c50ce833..b57ff16e5a 100644 --- a/crates/tracedecay-daemon-control/src/service/tests.rs +++ b/crates/tracedecay-daemon-control/src/service/tests.rs @@ -852,6 +852,27 @@ fn systemd_unit_quotes_exec_start_paths_that_systemd_would_misparse() { ); } +/// The daemon indexes on a worker pool sized to the machine. Capping glibc's +/// malloc arenas in the unit made every allocation on every worker contend +/// for two locks (60% of daemon CPU in the arena futex on a 20-worker host) +/// without bounding RSS, so the unit leaves the allocator alone. +#[test] +fn systemd_unit_does_not_cap_malloc_arenas() { + let spec = DaemonServiceSpec { + tracedecay_bin: PathBuf::from("/usr/local/bin/tracedecay"), + socket_path: PathBuf::from("/run/user/1000/tracedecay.sock"), + data_dir_override: None, + remote_tls: None, + }; + + let unit = spec.render_systemd_user_unit().expect("systemd unit"); + + assert!( + !unit.contains("MALLOC_ARENA_MAX"), + "the service unit must not serialize the index workers on a malloc arena cap, got:\n{unit}" + ); +} + #[test] fn systemd_socket_read_back_returns_none_for_unterminated_exec_start_quote() { let unit = "[Service]\nExecStart=/usr/bin/tracedecay daemon run --socket \"/run/unterminated\n";