From 1d3726b3b15ff8fe26380bd375a6784524cd78d2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 29 Oct 2026 05:28:10 -0400 Subject: [PATCH] drm/asahi: g17: admit a context's ioctls to work states in submission order Each engine admits a context's ioctls in submission order, but the render and compute engines advance independently. Under deep mixed load, render admission takes work-state nodes for ioctls far ahead of an older ioctl that is still waiting for its turn on the compute queue. gpubench-stress queue.depth keeps 128 compute and 64 render submissions in flight on one queue, each render followed by a compute copy. On a Neo (A18 Pro), every completion of the context stops within 0.5-4 s: compute and render-pair stamps stop together, newly published compute never runs, and the client loses the device. Capping the context's live nodes at 32, 48 or 56 does not prevent this. Native macOS on the same machine and firmware runs the same load (one MTLCommandQueue, 128 compute and 64 render command buffers in flight, each render followed by a compute copy) for 60 s without an error. Metal schedules a queue's command buffers in commit order across engines, with at most 36 between scheduling and completion. An ioctl now takes its node only while at most 8 older ioctls of its context have neither taken one nor left without one. Native scheduling happens at commit, on one thread. Here each engine admits from its own scheduler, so a strictly ordered turn costs a scheduler round trip whenever the other engine's ioctl is ready first; that slowed a windowed Vulkan client. Under the queue.depth load, run-ahead limits up to 16 held and 32 did not, so 8 leaves a factor of two. The turn is tracked under a context mutex taken inside the work-storage lock. Ioctls that fail or are cancelled before taking a node are remembered within 1024 ordinals of the turn. One that leaves further ahead makes the context's admission unordered, as before, rather than stall it. A refused publisher waits on its engine's admission domain, and the turn moving past a refusal wakes both domains. Contexts whose ioctls are never refused, such as single-engine ones, take no wake-ups from this. The last-node rule stays: render ioctls still hold nodes while their copies wait in the compute queue. Link: https://github.com/aurora-silicon/gpubench/issues/5 Signed-off-by: DjDeveloperr (cherry picked from commit c510e9d9ca38aa73c0affe88c786d9ae09a5ce93) --- drivers/gpu/drm/asahi/g17/context.rs | 43 ++++- drivers/gpu/drm/asahi/g17/context/identity.rs | 160 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 11 +- 3 files changed, 210 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index a40358d1be3c6f..0262ea10853983 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; -use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use super::job::Engine; use kernel::{ new_mutex, @@ -128,6 +128,11 @@ pub(crate) struct Context { states: WorkStates, /// Signalled when a work state returns; publishers that found none wait on it. state_returns: Arc, + /// The domain refused compute publishers wait on; signalled with `state_returns` when the + /// admission turn moves on. + compute_returns: Arc, + /// The order in which this context's ioctls take work-state nodes. + order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, @@ -139,6 +144,10 @@ pub(crate) struct Context { policy: Policy, } impl Context { + #[expect( + clippy::too_many_arguments, + reason = "A context takes its allocators, identities and both admission domains." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -147,6 +156,7 @@ impl Context { owner_pid: u32, policy: Policy, state_returns: Arc, + compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -195,6 +205,8 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + let order = + Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, @@ -208,6 +220,8 @@ impl Context { owner_pid, states: WorkStates::new(), state_returns, + compute_returns, + order, work, work_base, render, @@ -399,18 +413,23 @@ pub(crate) struct WorkStateLease { word: AtomicU32, /// Index of the engine whose admission took the node, valid once `word` is set. engine: AtomicU8, + /// The ioctl's place in its context's submission order. + ordinal: u64, } impl WorkStateLease { pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { + let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), engine: AtomicU8::new(0), + ordinal, } } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: none - /// may be taken now (see `identity::may_take`); a later return makes one available. + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many + /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); if current != FREE { @@ -423,6 +442,10 @@ impl WorkStateLease { if current != FREE { return Ok(current); } + let mut order = self.context.order.lock(); + if !order.admits(self.ordinal) { + return Err(EBUSY); + } let engine = engine.index(); let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; if let Err(error) = work.initialize(candidate) { @@ -431,6 +454,12 @@ impl WorkStateLease { } self.engine.store(engine as u8, Ordering::Relaxed); self.word.store(candidate, Ordering::Release); + let wake = order.admit(self.ordinal); + drop(order); + drop(work); + if wake { + self.context.pass_turn(); + } Ok(candidate) } /// The domain a publisher samples before acquire() and waits on after EBUSY. @@ -478,6 +507,8 @@ impl Drop for WorkStateLease { let word = self.word(); if word != FREE { self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); + } else if self.context.order.lock().leave(self.ordinal) { + self.context.pass_turn(); } } } @@ -490,6 +521,12 @@ impl Context { self.state_returns.notify(); } } + /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a + /// refusal means one that sampled before its refused acquire() never sleeps through it. + fn pass_turn(&self) { + self.state_returns.notify(); + self.compute_returns.notify(); + } } impl QosIds { diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 9162e3e6bc4a4c..7586f8474e602f 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -77,6 +77,101 @@ impl WorkStates { } } +/// Ordinals past the turn remembered for ioctls that took a node or left early. +pub(super) const ORDER_WINDOW: u64 = 1024; + +/// How far past the oldest waiting ioctl an ioctl may take its node. +pub(super) const RUN_AHEAD: u64 = 8; + +/// The order in which a context's ioctls take work-state nodes. +/// +/// Each engine admits a context's ioctls in submission order, but the engines advance +/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still +/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render +/// ioctls in flight on one queue) the firmware then stops completing all of the context's work +/// within seconds. Native macOS schedules a queue's command buffers in commit order across +/// engines, at most 36 between scheduling and completion, and the same load runs there without +/// failure. +/// +/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have +/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; +/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler +/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that +/// without letting one engine get far ahead of the other. +pub(super) struct Order { + /// The next ordinal to hand out. + next: u64, + /// The oldest ordinal that has neither taken a node nor left. + turn: u64, + /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + done: [u64; (ORDER_WINDOW / 64) as usize], + /// Set once an ioctl left too far past the turn to remember; admission is unordered after. + unordered: bool, + /// An ioctl was refused since the turn last moved: its publisher waits for the next move. + refused: bool, +} +impl Order { + pub(super) const fn new() -> Self { + Self { + next: 0, + turn: 0, + done: [0; (ORDER_WINDOW / 64) as usize], + unordered: false, + refused: false, + } + } + /// The ordinal of a new ioctl, in submission order. + pub(super) fn assign(&mut self) -> u64 { + let ordinal = self.next; + self.next += 1; + ordinal + } + /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the + /// turn next moves, so that move wakes the refused publisher. + pub(super) fn admits(&mut self, ordinal: u64) -> bool { + let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + self.refused |= !admits; + admits + } + /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + pub(super) fn admit(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). + /// Returns whether a refused ioctl may now be admitted. + pub(super) fn leave(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + fn finish(&mut self, ordinal: u64) -> bool { + if self.unordered || ordinal < self.turn { + return false; + } + if ordinal == self.turn { + self.pass(); + return core::mem::take(&mut self.refused); + } + if ordinal - self.turn >= ORDER_WINDOW { + self.unordered = true; + return core::mem::take(&mut self.refused); + } + let bit = ordinal % ORDER_WINDOW; + self.done[(bit / 64) as usize] |= 1 << (bit % 64); + false + } + /// Moves the turn past the current ordinal and every following one already done. + fn pass(&mut self) { + loop { + self.turn += 1; + let bit = self.turn % ORDER_WINDOW; + let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); + if self.done[word] & mask == 0 { + break; + } + self.done[word] &= !mask; + } + } +} + /// Two stable logical ordinals, independent of physical pair numbering. pub(super) struct RenderSlots { pairs: [Option; 2], @@ -154,6 +249,71 @@ mod tests { assert!(states.allocate(1).is_some()); } + #[test] + fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + let mut order = Order::new(); + let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); + let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); + // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. + for &ordinal in &ordinals[..=RUN_AHEAD as usize] { + assert!(order.admits(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // Ioctls ahead of the turn taking their nodes do not move it. + for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { + assert!(!order.admit(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // The oldest one moves the turn past everything already admitted, and wakes the refused. + assert!(order.admit(oldest)); + assert!(order.admits(last)); + let next = order.assign(); + assert!(order.admits(next)); + assert!(!order.admits(next + RUN_AHEAD + 1)); + } + + #[test] + fn ioctls_leaving_without_a_node_pass_the_turn() { + let mut order = Order::new(); + let (a, b, c) = (order.assign(), order.assign(), order.assign()); + let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); + let beyond = later[RUN_AHEAD as usize - 1]; + // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + assert!(!order.leave(b)); + assert!(!order.leave(c)); + assert!(!order.admits(beyond)); + // Once a takes its node, the turn passes b and c as well, and the refused one may retry. + assert!(order.admit(a)); + assert!(order.admits(beyond)); + // With nobody refused since, moving the turn wakes nobody; an ordinal already passed + // leaves without effect. + assert!(!order.leave(later[0])); + assert!(!order.leave(a)); + } + + #[test] + fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + let mut order = Order::new(); + let first = order.assign(); + let mut last = first; + for _ in 0..ORDER_WINDOW { + last = order.assign(); + } + assert!(!order.admits(last)); + assert!(order.leave(last)); + assert!(order.admits(last - 1)); + } + + #[test] + fn single_engine_admission_wakes_nobody() { + let mut order = Order::new(); + for _ in 0..3 * RUN_AHEAD { + let ordinal = order.assign(); + assert!(order.admits(ordinal)); + assert!(!order.admit(ordinal)); + } + } + #[test] fn releases_return_nodes_to_their_engine() { let states = WorkStates::new(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index eb0396745f3da6..0d1e390f19bb6a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -754,7 +754,16 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, + Context::new( + &alloc, + &pool, + &qos, + execution, + owner_pid, + policy, + shared.renders.clone(), + shared.progress.clone(), + )?, GFP_KERNEL, )?; let state = shared.state.lock();