From 3a3aea300f38b0914ce1de3530277a67755cb544 Mon Sep 17 00:00:00 2001 From: Nick Angelou Date: Thu, 19 Nov 2026 23:34:37 +0100 Subject: [PATCH] Upgrade scheduler to Ferroid 3.1.0 Adapt job ID generation to the v3 clock, async, and base32 APIs and refresh the lockfile. --- Cargo.lock | 27 +++++++++++++------ ballista/scheduler/Cargo.toml | 2 +- .../scheduler/src/scheduler_server/mod.rs | 14 +++++----- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7248d6f340..1662cc15b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1285,7 +1285,7 @@ dependencies = [ "datafusion", "datafusion-proto", "datafusion-substrait", - "ferroid", + "ferroid 3.1.0", "futures", "graphviz-rust", "http 1.5.0", @@ -3390,10 +3390,21 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" dependencies = [ - "futures", "portable-atomic", "rand 0.10.3", - "tokio", + "web-time", +] + +[[package]] +name = "ferroid" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c63948fb5874d42ea7562d7051437eb22260fccbb502e27a3688cfdae3b9b54" +dependencies = [ + "crossbeam-utils", + "portable-atomic", + "serde", + "tracing", "web-time", ] @@ -5419,9 +5430,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -5576,7 +5587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5597,7 +5608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad9c5423459fa2f729b8d9235fab810aa29628b171467744fc86d21460ad70d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.15.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7140,7 +7151,7 @@ dependencies = [ "docker_credential", "either", "etcetera", - "ferroid", + "ferroid 2.0.0", "futures", "http 1.5.0", "itertools 0.14.0", diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index 3e6fa82825..bc73192798 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -81,7 +81,7 @@ dashmap = { workspace = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-substrait = { workspace = true, optional = true } -ferroid = { version = "2.0", features = ["alloc", "std", "snowflake", "atomic", "base32", "async-tokio"] } +ferroid = { version = "3", features = ["alloc", "std", "snowflake", "atomic", "base32", "futures"] } futures = { workspace = true } graphviz-rust = { version = "0.9", optional = true } http = "1.4" diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs index 1a79b8d877..a6dc347f81 100644 --- a/ballista/scheduler/src/scheduler_server/mod.rs +++ b/ballista/scheduler/src/scheduler_server/mod.rs @@ -32,11 +32,11 @@ use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; -use ferroid::base32::Base32SnowExt; -use ferroid::futures::SnowflakeGeneratorAsyncTokioExt; +use ferroid::base32::Base32Ext; +use ferroid::futures::SnowflakeGeneratorAsyncExt; use ferroid::generator::AtomicSnowflakeGenerator; use ferroid::id::SnowflakeMastodonId; -use ferroid::time::MonotonicClock; +use ferroid::time::CachedClock; use tokio::sync::broadcast; use crate::cache::{contains_cache, strip_caches}; @@ -98,7 +98,7 @@ pub trait JobIdGenerator: Sync + Send { /// A monotonically increasing sortable snowflake job id generator /// which does not capture machine id as part result. pub(crate) struct DefaultJobGenerator { - generator: AtomicSnowflakeGenerator, + generator: AtomicSnowflakeGenerator, } impl Default for DefaultJobGenerator { @@ -107,7 +107,7 @@ impl Default for DefaultJobGenerator { // setups let generator = AtomicSnowflakeGenerator::new( 0, // machine id is hard-coded to 0 - MonotonicClock::<1>::default(), + CachedClock::default(), ); Self { generator } @@ -117,8 +117,8 @@ impl Default for DefaultJobGenerator { #[async_trait::async_trait] impl JobIdGenerator for DefaultJobGenerator { async fn next_id(&self) -> String { - let id = self.generator.next_id_async().await; - id.encode().to_string() + let id = self.generator.next_id_async(tokio::time::sleep).await; + id.encode().as_string() } }