From 427e1cd1af04de56d98aa82e7bf53f9241899a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Max?= Date: Fri, 13 Nov 2026 12:09:35 -0300 Subject: [PATCH 1/5] test(executor): pin scheduler connect, memory pool metrics and clean-up at startup Add characterization tests for start_executor_process ahead of splitting start_executor_process_inner into phases (#2597): the single connect attempt with scheduler_connect_timeout_seconds = 0, the memory pool size reported to ExecutorMemoryMetrics for a configured and an unbounded pool, and the shuffle clean-up: with job_data_clean_up_interval_seconds = 0 neither the periodic cleaner nor the clean-up at shutdown runs, and with a positive interval job data is removed on shutdown. The last two run a pull-based executor against an idle in-process scheduler and shut it down by taking its Flight port. Co-authored-by: Megabot --- ballista/executor/src/executor_process.rs | 285 ++++++++++++++++++++++ 1 file changed, 285 insertions(+) diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 04c453c487..9986492565 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -1450,15 +1450,33 @@ mod tests { use super::clean_shuffle_data_loop; use super::remove_job_data; use super::structure_executor_metadata; + use crate::metrics::ExecutorMemoryMetrics; use ballista_core::BALLISTA_PROTOCOL_VERSION; use ballista_core::JobId; + use ballista_core::config::TaskSchedulingPolicy; use ballista_core::ids::new_instance_id; use ballista_core::serde::protobuf::executor_resource::Resource; + use ballista_core::serde::protobuf::scheduler_grpc_server::{ + SchedulerGrpc, SchedulerGrpcServer, + }; + use ballista_core::serde::protobuf::{ + CancelJobParams, CancelJobResult, CleanJobDataParams, CleanJobDataResult, + CreateUpdateSessionParams, CreateUpdateSessionResult, ExecuteQueryParams, + ExecuteQueryResult, ExecutorStoppedParams, ExecutorStoppedResult, + GetJobMetricsParams, GetJobMetricsResult, GetJobStatusParams, GetJobStatusResult, + HeartBeatParams, HeartBeatResult, PollWorkParams, PollWorkResult, + RegisterExecutorParams, RegisterExecutorResult, RemoveSessionParams, + RemoveSessionResult, UpdateTaskStatusParams, UpdateTaskStatusResult, + }; use std::fs; use std::fs::File; use std::io::Write; use std::time::Duration; use tempfile::TempDir; + use tokio::net::TcpListener; + use tokio_stream::wrappers::TcpListenerStream; + use tonic::codegen::BoxStream; + use tonic::{Request, Response, Status}; #[test] fn executor_metadata_reflects_process_configuration() { @@ -1570,6 +1588,273 @@ mod tests { drop(listener); } + /// Reserves a local port and releases it, so nothing listens there. + fn unused_local_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() + } + + type GrpcResult = std::result::Result, Status>; + + fn not_called() -> GrpcResult { + Err(Status::unimplemented("not called by a pull-based executor")) + } + + /// A scheduler that answers what a pull-based executor asks of it, and + /// never has any work to hand out. + struct IdleScheduler; + + #[tonic::async_trait] + impl SchedulerGrpc for IdleScheduler { + type ExecuteQueryPushStream = BoxStream; + + async fn poll_work( + &self, + _: Request, + ) -> GrpcResult { + Ok(Response::new(PollWorkResult { + tasks: vec![], + jobs_to_clean: vec![], + })) + } + + async fn heart_beat_from_executor( + &self, + _: Request, + ) -> GrpcResult { + Ok(Response::new(HeartBeatResult::default())) + } + + async fn executor_stopped( + &self, + _: Request, + ) -> GrpcResult { + Ok(Response::new(ExecutorStoppedResult::default())) + } + + async fn register_executor( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn update_task_status( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn create_update_session( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn remove_session( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn execute_query_push( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn execute_query( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn get_job_status( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn get_job_metrics( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn cancel_job( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + + async fn clean_job_data( + &self, + _: Request, + ) -> GrpcResult { + not_called() + } + } + + /// Runs a pull-based executor against an idle scheduler, with the given + /// clean-up interval and `work_dir` as its work directory, until it shuts + /// itself down. The shutdown comes from its Flight port being taken, which + /// stops a service right after startup; it then goes through the normal + /// graceful shutdown path, which runs to the end with the scheduler. + /// + /// This sets the process-wide `TERMINATING` flag, which no other unit test + /// reads. + async fn run_executor_until_flight_port_is_taken( + job_data_clean_up_interval_seconds: u64, + work_dir: &Path, + ) { + let scheduler = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let scheduler_port = scheduler.local_addr().unwrap().port(); + tokio::spawn( + tonic::transport::Server::builder() + .add_service(SchedulerGrpcServer::new(IdleScheduler)) + .serve_with_incoming(TcpListenerStream::new(scheduler)), + ); + let flight_port_holder = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + + let config = Arc::new(ExecutorProcessConfig { + scheduler_host: "127.0.0.1".to_string(), + scheduler_port, + scheduler_connect_timeout_seconds: 5, + bind_host: "127.0.0.1".to_string(), + port: flight_port_holder.local_addr().unwrap().port(), + task_scheduling_policy: TaskSchedulingPolicy::PullStaged, + work_dir: Some(work_dir.to_str().unwrap().to_string()), + job_data_clean_up_interval_seconds, + ..ExecutorProcessConfig::default() + }); + + tokio::time::timeout( + Duration::from_secs(30), + super::start_executor_process(config), + ) + .await + .expect("the executor did not shut down") + .expect("the executor failed instead of shutting down gracefully"); + drop(flight_port_holder); + } + + /// A job directory with a file in it, as the shuffle writer leaves one. + fn create_job_dir(work_dir: &Path) -> PathBuf { + let job_dir = work_dir.join("job_id"); + fs::create_dir(&job_dir).unwrap(); + File::create(job_dir.join("data.arrow")) + .unwrap() + .write_all(b"data") + .unwrap(); + job_dir + } + + #[tokio::test] + async fn clean_up_interval_zero_disables_the_cleaner_and_the_shutdown_clean_up() { + let work_dir = TempDir::new().unwrap(); + let job_dir = create_job_dir(work_dir.path()); + + run_executor_until_flight_port_is_taken(0, work_dir.path()).await; + + assert!( + job_dir.exists(), + "job data was cleaned although the clean-up interval is 0" + ); + } + + #[tokio::test] + async fn positive_clean_up_interval_cleans_job_data_on_shutdown() { + let work_dir = TempDir::new().unwrap(); + let job_dir = create_job_dir(work_dir.path()); + + run_executor_until_flight_port_is_taken(1, work_dir.path()).await; + + assert!( + !job_dir.exists(), + "job data was not cleaned on shutdown with a clean-up interval of 1" + ); + } + + #[tokio::test] + async fn scheduler_connect_zero_timeout_makes_a_single_attempt() { + let config = Arc::new(ExecutorProcessConfig { + scheduler_host: "127.0.0.1".to_string(), + scheduler_port: unused_local_port(), + scheduler_connect_timeout_seconds: 0, + ..ExecutorProcessConfig::default() + }); + + let result = tokio::time::timeout( + Duration::from_secs(5), + super::start_executor_process(config), + ) + .await + .expect("executor kept retrying with scheduler_connect_timeout_seconds = 0"); + + let err = result.unwrap_err(); + assert!( + err.to_string().contains("Could not connect to scheduler"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn configured_memory_pool_size_is_reported_to_metrics() { + let metrics = ExecutorMemoryMetrics::default(); + let config = Arc::new(ExecutorProcessConfig { + scheduler_host: "127.0.0.1".to_string(), + scheduler_port: unused_local_port(), + scheduler_connect_timeout_seconds: 0, + memory_pool_size: Some(64 * 1024 * 1024), + ..ExecutorProcessConfig::default() + }); + + let result = tokio::time::timeout( + Duration::from_secs(5), + super::start_executor_process_with_memory_metrics(config, metrics.clone()), + ) + .await + .expect("executor did not fail fast without a scheduler"); + assert!(result.is_err()); + + let usage = metrics.snapshot().expect("pool size was never recorded"); + assert_eq!(usage.pool_size, Some(64 * 1024 * 1024)); + } + + #[tokio::test] + async fn unbounded_memory_pool_is_reported_to_metrics() { + let metrics = ExecutorMemoryMetrics::default(); + let config = Arc::new(ExecutorProcessConfig { + scheduler_host: "127.0.0.1".to_string(), + scheduler_port: unused_local_port(), + scheduler_connect_timeout_seconds: 0, + memory_pool_size: Some(0), + ..ExecutorProcessConfig::default() + }); + + let result = tokio::time::timeout( + Duration::from_secs(5), + super::start_executor_process_with_memory_metrics(config, metrics.clone()), + ) + .await + .expect("executor did not fail fast without a scheduler"); + assert!(result.is_err()); + + let usage = metrics.snapshot().expect("pool size was never recorded"); + assert_eq!(usage.pool_size, None); + } + #[test] fn default_executor_id_is_uuid() { let config = ExecutorProcessConfig::default(); From fdbf50671b3356e0ad3c93f9c6938c70161148b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Max?= Date: Fri, 13 Nov 2026 12:11:35 -0300 Subject: [PATCH 2/5] refactor(executor): extract resolve_vcores from start_executor_process_inner Part of #2597. Moves the vcores resolution verbatim; behaviour is unchanged. Co-authored-by: Megabot --- ballista/executor/src/executor_process.rs | 41 +++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 9986492565..f8374844c5 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -616,6 +616,17 @@ pub async fn start_executor_process_with_memory_metrics( start_executor_process_inner(opt, Some(memory_metrics)).await } +/// Number of vcores the executor advertises: the configured value, or all +/// available cores when it is `0`. +fn resolve_vcores(opt: &ExecutorProcessConfig) -> usize { + if opt.vcores == 0 { + // use all available cores if no concurrency level is specified + std::thread::available_parallelism().unwrap().get() + } else { + opt.vcores + } +} + async fn start_executor_process_inner( opt: Arc, memory_metrics: Option, @@ -642,12 +653,7 @@ async fn start_executor_process_inner( )); }; - let vcores = if opt.vcores == 0 { - // use all available cores if no concurrency level is specified - std::thread::available_parallelism().unwrap().get() - } else { - opt.vcores - }; + let vcores = resolve_vcores(&opt); let task_scheduling_policy = opt.task_scheduling_policy; let executor_id = opt.executor_id.clone(); info!( @@ -1588,6 +1594,29 @@ mod tests { drop(listener); } + #[test] + fn resolve_vcores_zero_uses_available_parallelism() { + let config = ExecutorProcessConfig { + vcores: 0, + ..ExecutorProcessConfig::default() + }; + + assert_eq!( + super::resolve_vcores(&config), + std::thread::available_parallelism().unwrap().get() + ); + } + + #[test] + fn resolve_vcores_explicit_is_kept() { + let config = ExecutorProcessConfig { + vcores: 3, + ..ExecutorProcessConfig::default() + }; + + assert_eq!(super::resolve_vcores(&config), 3); + } + /// Reserves a local port and releases it, so nothing listens there. fn unused_local_port() -> u16 { std::net::TcpListener::bind("127.0.0.1:0") From 05dbf1fbe7980605066503f79ceb07807586e3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Max?= Date: Fri, 13 Nov 2026 12:12:47 -0300 Subject: [PATCH 3/5] refactor(executor): extract spawn_shuffle_cleaner from start_executor_process_inner Part of #2597. Moves the periodic shuffle cleaner task verbatim; behaviour is unchanged. Adds a test that it removes expired job dirs and stops on shutdown. Co-authored-by: Megabot --- ballista/executor/src/executor_process.rs | 98 +++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index f8374844c5..5f613a31cf 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -627,6 +627,40 @@ fn resolve_vcores(opt: &ExecutorProcessConfig) -> usize { } } +/// Spawns the periodic shuffle data cleaner, which stops when the shutdown +/// notification is received. +fn spawn_shuffle_cleaner( + shutdown_notification: &ShutdownNotifier, + work_dir: &str, + clean_up_interval_seconds: u64, + job_data_ttl_seconds: u64, +) { + let mut interval_time = + time::interval(Duration::from_secs(clean_up_interval_seconds)); + let cleaner_work_dir = work_dir.to_owned(); + + let mut shuffle_cleaner_shutdown = shutdown_notification.subscribe_for_shutdown(); + let shuffle_cleaner_complete = shutdown_notification.shutdown_complete_tx.clone(); + + tokio::spawn(async move { + // As long as the shutdown notification has not been received + while !shuffle_cleaner_shutdown.is_shutdown() { + tokio::select! { + _ = interval_time.tick() => { + if let Err(e) = clean_shuffle_data_loop(&cleaner_work_dir, job_data_ttl_seconds).await + { + error!("Ballista executor fail to clean_shuffle_data {e:?}") + } + }, + _ = shuffle_cleaner_shutdown.recv() => { + drop(shuffle_cleaner_complete); + return; + } + }; + } + }); +} + async fn start_executor_process_inner( opt: Arc, memory_metrics: Option, @@ -912,30 +946,12 @@ async fn start_executor_process_inner( let clean_job_data_on_shutdown = opt.job_data_clean_up_interval_seconds > 0; if clean_job_data_on_shutdown { - let mut interval_time = - time::interval(Duration::from_secs(opt.job_data_clean_up_interval_seconds)); - let cleaner_work_dir = work_dir.clone(); - - let mut shuffle_cleaner_shutdown = shutdown_notification.subscribe_for_shutdown(); - let shuffle_cleaner_complete = shutdown_notification.shutdown_complete_tx.clone(); - - tokio::spawn(async move { - // As long as the shutdown notification has not been received - while !shuffle_cleaner_shutdown.is_shutdown() { - tokio::select! { - _ = interval_time.tick() => { - if let Err(e) = clean_shuffle_data_loop(&cleaner_work_dir, job_data_ttl_seconds).await - { - error!("Ballista executor fail to clean_shuffle_data {e:?}") - } - }, - _ = shuffle_cleaner_shutdown.recv() => { - drop(shuffle_cleaner_complete); - return; - } - }; - } - }); + spawn_shuffle_cleaner( + &shutdown_notification, + &work_dir, + opt.job_data_clean_up_interval_seconds, + job_data_ttl_seconds, + ); } let mut service_handlers: FuturesUnordered>> = @@ -1457,6 +1473,7 @@ mod tests { use super::remove_job_data; use super::structure_executor_metadata; use crate::metrics::ExecutorMemoryMetrics; + use crate::shutdown::ShutdownNotifier; use ballista_core::BALLISTA_PROTOCOL_VERSION; use ballista_core::JobId; use ballista_core::config::TaskSchedulingPolicy; @@ -1617,6 +1634,39 @@ mod tests { assert_eq!(super::resolve_vcores(&config), 3); } + #[tokio::test] + async fn shuffle_cleaner_removes_expired_job_dirs_and_stops_on_shutdown() { + let work_dir = TempDir::new().unwrap(); + let job_dir = work_dir.path().join("job_id"); + fs::create_dir(&job_dir).unwrap(); + File::create(job_dir.join("tmp.csv")) + .expect("creating temp file") + .write_all(b"data") + .expect("writing data"); + + let mut notifier = ShutdownNotifier::new(); + super::spawn_shuffle_cleaner(¬ifier, work_dir.path().to_str().unwrap(), 1, 1); + + tokio::time::timeout(Duration::from_secs(10), async { + while job_dir.exists() { + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("expired job dir was never cleaned"); + + // The cleaner holds a completion sender until it sees the shutdown + // signal, so the receiver only completes once the cleaner has stopped. + let _ = notifier.notify_shutdown.send(()); + drop(notifier.shutdown_complete_tx); + tokio::time::timeout( + Duration::from_secs(5), + notifier.shutdown_complete_rx.recv(), + ) + .await + .expect("shuffle cleaner did not stop on shutdown"); + } + /// Reserves a local port and releases it, so nothing listens there. fn unused_local_port() -> u16 { std::net::TcpListener::bind("127.0.0.1:0") From 7b52b08d89024ebdaedab77afd0b7be2e48e84f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Max?= Date: Fri, 13 Nov 2026 12:15:31 -0300 Subject: [PATCH 4/5] refactor(executor): extract run_until_shutdown from start_executor_process_inner Part of #2597. Moves the service/signal/stop-request select verbatim; behaviour is unchanged. Adds tests for the stop-request and failed-service outcomes. Co-authored-by: Megabot --- ballista/executor/src/executor_process.rs | 86 ++++++++++++++++++----- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 5f613a31cf..28e3424404 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -1041,22 +1041,8 @@ async fn start_executor_process_inner( let tasks_drained = TasksDrainedFuture(executor.clone()); - // Concurrently run the service checking and listen for the `shutdown` signal and wait for the stop request coming. - // The check_services runs until an error is encountered, so under normal circumstances, this `select!` statement runs - // until the `shutdown` signal is received or a stop request is coming. - let (notify_scheduler, stop_reason) = tokio::select! { - service_val = check_services(&mut service_handlers) => { - let msg = format!("executor services stopped with reason {service_val:?}"); - info!("{msg:?}"); - (true, msg) - }, - signal = shutdown_signal() => { - (true, format!("executor received {signal}.")) - }, - _ = stop_recv.recv() => { - (false, "".to_string()) - }, - }; + let (notify_scheduler, stop_reason) = + run_until_shutdown(&mut service_handlers, &mut stop_recv).await; // Set status to fenced info!("setting executor to TERMINATING status"); @@ -1173,6 +1159,30 @@ async fn flight_server_task( }) } +/// Waits until a service fails or finishes, a shutdown signal arrives, or a stop +/// request is received. Returns whether to notify the scheduler and the reason. +async fn run_until_shutdown( + service_handlers: &mut FuturesUnordered>>, + stop_recv: &mut mpsc::Receiver, +) -> (bool, String) { + // Concurrently run the service checking and listen for the `shutdown` signal and wait for the stop request coming. + // The check_services runs until an error is encountered, so under normal circumstances, this `select!` statement runs + // until the `shutdown` signal is received or a stop request is coming. + tokio::select! { + service_val = check_services(service_handlers) => { + let msg = format!("executor services stopped with reason {service_val:?}"); + info!("{msg:?}"); + (true, msg) + }, + signal = shutdown_signal() => { + (true, format!("executor received {signal}.")) + }, + _ = stop_recv.recv() => { + (false, "".to_string()) + }, + } +} + // Check the status of long running services async fn check_services( service_handlers: &mut FuturesUnordered>>, @@ -1477,6 +1487,7 @@ mod tests { use ballista_core::BALLISTA_PROTOCOL_VERSION; use ballista_core::JobId; use ballista_core::config::TaskSchedulingPolicy; + use ballista_core::error::BallistaError; use ballista_core::ids::new_instance_id; use ballista_core::serde::protobuf::executor_resource::Resource; use ballista_core::serde::protobuf::scheduler_grpc_server::{ @@ -1491,6 +1502,7 @@ mod tests { RegisterExecutorParams, RegisterExecutorResult, RemoveSessionParams, RemoveSessionResult, UpdateTaskStatusParams, UpdateTaskStatusResult, }; + use futures::stream::FuturesUnordered; use std::fs; use std::fs::File; use std::io::Write; @@ -1667,6 +1679,48 @@ mod tests { .expect("shuffle cleaner did not stop on shutdown"); } + #[tokio::test] + async fn stop_request_ends_the_wait_without_notifying_the_scheduler() { + let mut handlers = FuturesUnordered::new(); + handlers.push(tokio::spawn(async { + std::future::pending::>().await + })); + let (stop_send, mut stop_recv) = tokio::sync::mpsc::channel::(10); + stop_send.send(true).await.unwrap(); + + let outcome = tokio::time::timeout( + Duration::from_secs(5), + super::run_until_shutdown(&mut handlers, &mut stop_recv), + ) + .await + .expect("a stop request did not end the wait"); + + assert_eq!(outcome, (false, String::new())); + } + + #[tokio::test] + async fn failed_service_ends_the_wait_and_notifies_the_scheduler() { + let mut handlers = FuturesUnordered::new(); + handlers.push(tokio::spawn(async { + Err::<(), BallistaError>(BallistaError::General("boom".to_string())) + })); + let (_stop_send, mut stop_recv) = tokio::sync::mpsc::channel::(10); + + let (notify_scheduler, reason) = tokio::time::timeout( + Duration::from_secs(5), + super::run_until_shutdown(&mut handlers, &mut stop_recv), + ) + .await + .expect("a failed service did not end the wait"); + + assert!(notify_scheduler); + assert!( + reason.starts_with("executor services stopped with reason Err("), + "unexpected reason: {reason}" + ); + assert!(reason.contains("boom"), "unexpected reason: {reason}"); + } + /// Reserves a local port and releases it, so nothing listens there. fn unused_local_port() -> u16 { std::net::TcpListener::bind("127.0.0.1:0") From 6c31cff1caedc1dae80e8457016f3a180ab64888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Max?= Date: Fri, 13 Nov 2026 12:17:21 -0300 Subject: [PATCH 5/5] refactor(executor): extract resolve_memory_pool_policy from start_executor_process_inner Part of #2597. Moves the memory pool resolution and its metrics wiring verbatim; behaviour is unchanged. The metrics handle is passed as Option<&ExecutorMemoryMetrics>. Covered by the memory pool metrics tests. Co-authored-by: Megabot --- ballista/executor/src/executor_process.rs | 110 ++++++++++++---------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs index 28e3424404..708c8476d9 100644 --- a/ballista/executor/src/executor_process.rs +++ b/ballista/executor/src/executor_process.rs @@ -661,6 +661,66 @@ fn spawn_shuffle_cleaner( }); } +/// Resolves the executor memory pool budget into the per-task pool policy, +/// recording the budget in `memory_metrics` when they are enabled. +fn resolve_memory_pool_policy( + opt: &ExecutorProcessConfig, + vcores: usize, + memory_metrics: Option<&ExecutorMemoryMetrics>, +) -> ballista_core::error::Result { + let fraction = opt.memory_pool_fraction; + let pool_policy: MemoryPoolPolicy = + match detect_pool(memory_budget_from_cli(opt.memory_pool_size, fraction)) { + ResolvedPool::Bounded { bytes, source } => { + if let Some(metrics) = memory_metrics { + metrics.set_pool_size(Some(bytes)); + } + let per_vcore = bytes / vcores as u64; + let source_desc = match source { + PoolSource::Configured => { + "configured via --memory-pool-size".to_string() + } + PoolSource::AutoCgroup => { + format!("auto: {:.0}% of cgroup memory limit", fraction * 100.0) + } + PoolSource::AutoHost => { + format!("auto: {:.0}% of host memory", fraction * 100.0) + } + }; + info!( + "Executor memory pool: {} ({source_desc}), \ + split across {vcores} vcores ({}/vcore)", + bytesize::ByteSize::b(bytes), + bytesize::ByteSize::b(per_vcore), + ); + memory_pool_policy(bytes, vcores)? + } + ResolvedPool::Unbounded(reason) => { + if let Some(metrics) = memory_metrics { + metrics.set_pool_size(None); + } + match reason { + UnboundedReason::Explicit => { + info!("Executor memory pool: unbounded (--memory-pool-size 0)") + } + UnboundedReason::Undetected => warn!( + "Executor memory pool: unbounded (could not detect host or \ + cgroup memory limit; set --memory-pool-size to enable spilling)" + ), + } + identity_pool_policy() + } + }; + + let pool_policy = if let Some(metrics) = memory_metrics.cloned() { + observe_memory_pools(pool_policy, metrics) + } else { + pool_policy + }; + + Ok(pool_policy) +} + async fn start_executor_process_inner( opt: Arc, memory_metrics: Option, @@ -728,55 +788,7 @@ async fn start_executor_process_inner( None => base_runtime_producer, }; - let fraction = opt.memory_pool_fraction; - let pool_policy: MemoryPoolPolicy = - match detect_pool(memory_budget_from_cli(opt.memory_pool_size, fraction)) { - ResolvedPool::Bounded { bytes, source } => { - if let Some(metrics) = &memory_metrics { - metrics.set_pool_size(Some(bytes)); - } - let per_vcore = bytes / vcores as u64; - let source_desc = match source { - PoolSource::Configured => { - "configured via --memory-pool-size".to_string() - } - PoolSource::AutoCgroup => { - format!("auto: {:.0}% of cgroup memory limit", fraction * 100.0) - } - PoolSource::AutoHost => { - format!("auto: {:.0}% of host memory", fraction * 100.0) - } - }; - info!( - "Executor memory pool: {} ({source_desc}), \ - split across {vcores} vcores ({}/vcore)", - bytesize::ByteSize::b(bytes), - bytesize::ByteSize::b(per_vcore), - ); - memory_pool_policy(bytes, vcores)? - } - ResolvedPool::Unbounded(reason) => { - if let Some(metrics) = &memory_metrics { - metrics.set_pool_size(None); - } - match reason { - UnboundedReason::Explicit => { - info!("Executor memory pool: unbounded (--memory-pool-size 0)") - } - UnboundedReason::Undetected => warn!( - "Executor memory pool: unbounded (could not detect host or \ - cgroup memory limit; set --memory-pool-size to enable spilling)" - ), - } - identity_pool_policy() - } - }; - - let pool_policy = if let Some(metrics) = memory_metrics.clone() { - observe_memory_pools(pool_policy, metrics) - } else { - pool_policy - }; + let pool_policy = resolve_memory_pool_policy(&opt, vcores, memory_metrics.as_ref())?; // Combined producer preserving the current per-task behavior: build a fresh // base env, then apply the pool policy. Used by `Executor::produce_runtime`