From fd2a0321db11572ac201663697c83c9330ef65a3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:17:25 -0400 Subject: [PATCH 1/9] driver core: track devres node membership for direct removal Rust Devres drops remove an exact node, but devres_node_remove scans the entire device list to find it. Repeated removal of older GPU SG-table resources therefore does quadratic work while holding devres_lock. After large concurrent GPU workloads this keeps exiting clients CPU-bound for roughly a minute. Track the device-list owner in each node and remove a matching node directly under the device lock. Clear membership on every detach path, including replacement, ordinary and action removal, group removal, and moves to the private release-all/group callback list. A concurrent drop must leave nodes already owned by that release path alone. Preserve callback ordering and lifetime handling. Use single-copy membership accesses so a simultaneous comparison using a different device does not race through an ordinary pointer access. The membership field adds one pointer per resource node. Source-derived ASan/UBSan controls cover the removal APIs, replacement, open/closed/nested groups, private-list ownership and 1000 concurrent removal/detachment/different-device lookup races. Retaining stale private list ownership makes the negative control fail. The combined series passes Neo G17P concurrent GPU stress, expiry cycles and retained-VM context churn. Four clients performing 30 seconds of checked compute work finish and exit in 31.46 seconds total, versus 86.97 seconds with the same compact target layout before this correction. Signed-off-by: DjDeveloperr --- drivers/base/base.h | 2 ++ drivers/base/devres.c | 29 ++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/base/base.h b/drivers/base/base.h index a5b7abc10ff02..d16026fdb2ef0 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -222,6 +222,8 @@ typedef void (*dr_node_free_t)(struct devres_node *node); struct devres_node { struct list_head entry; + /* Device-list membership, cleared under devres_lock before detachment. */ + struct device *owner; dr_node_release_t release; dr_node_free_t free_node; const char *name; diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 9d9842fc5a19f..26e5518b6482b 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -41,6 +41,7 @@ void devres_node_init(struct devres_node *node, dr_node_free_t free_node) { INIT_LIST_HEAD(&node->entry); + WRITE_ONCE(node->owner, NULL); node->release = release; node->free_node = free_node; } @@ -156,6 +157,7 @@ static void add_dr(struct device *dev, struct devres_node *node) devres_log(dev, node, "ADD"); BUG_ON(!list_empty(&node->entry)); list_add_tail(&node->entry, &dev->devres_head); + WRITE_ONCE(node->owner, dev); } static void replace_dr(struct device *dev, @@ -164,6 +166,8 @@ static void replace_dr(struct device *dev, devres_log(dev, old, "REPLACE"); BUG_ON(!list_empty(&new->entry)); list_replace(&old->entry, &new->entry); + WRITE_ONCE(old->owner, NULL); + WRITE_ONCE(new->owner, dev); } /** @@ -367,18 +371,14 @@ EXPORT_SYMBOL_GPL(devres_get); bool devres_node_remove(struct device *dev, struct devres_node *node) { - struct devres_node *__node; - guard(spinlock_irqsave)(&dev->devres_lock); - list_for_each_entry_reverse(__node, &dev->devres_head, entry) { - if (__node == node) { - list_del_init(&node->entry); - devres_log(dev, node, "REM"); - return true; - } - } - - return false; + /* A release-all/group pass may own a detached node on its private list. */ + if (READ_ONCE(node->owner) != dev) + return false; + list_del_init(&node->entry); + WRITE_ONCE(node->owner, NULL); + devres_log(dev, node, "REM"); + return true; } /** @@ -405,6 +405,7 @@ void *devres_remove(struct device *dev, dr_release_t release, dr = find_dr(dev, release, match, match_data); if (dr) { list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr->data; } @@ -500,6 +501,7 @@ static int remove_nodes(struct device *dev, if (&node->entry == first) first = first->next; list_move_tail(&node->entry, todo); + WRITE_ONCE(node->owner, NULL); cnt++; } } @@ -530,7 +532,9 @@ static int remove_nodes(struct device *dev, * nodes are always before both. */ list_move_tail(&grp->node[0].entry, todo); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); } } @@ -685,7 +689,9 @@ void devres_remove_group(struct device *dev, void *id) grp = find_group(dev, id); if (grp) { list_del_init(&grp->node[0].entry); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); devres_log(dev, &grp->node[0], "REM"); } else WARN_ON(1); @@ -849,6 +855,7 @@ static struct devres_action *remove_action(struct device *dev, return ERR_PTR(-ENOENT); list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr; From 833c6ed10ffc2ac2d38b7bbb0d85dcc2b8e171df Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:17:25 -0400 Subject: [PATCH 2/9] drm/asahi: reduce minimum G17 target record backing The target-record stride already accounts for its computed layout and UAT alignment, but its 256 KiB minimum wastes backing on smaller targets. Use one 32 KiB TVB page as the minimum instead, keeping the computed used size, consumer bounds, growth handling and all sixteen generations unchanged. Queue capacities, batching and synchronization policy are unchanged. Source-derived controls validate 388800 layouts against the original capacity and consumer bounds; 114426 need less backing. At forty held GPU clients with 124 target objects on Neo, allocation extents fall from 272236544 to 44695552 bytes, saving 217 MiB for that workload. The compact layout passes changing-pixel and layered readbacks, target growth, concurrent render/compute/mixed stress, Vulkan multisample checks, compute pool expiry and 2000 retained-VM contexts. Brilliant Diamond also reaches its language menu around 30 FPS in a bounded emulator run; broader gameplay and application performance validation remain separate. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/hw/t8140.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 41f04be56e28d..62282b2635b8a 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -273,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; From 145361fdc90e82f4bddf6234ec46d0e023f41f86 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:19:26 -0400 Subject: [PATCH 3/9] drm/asahi: order G17 queue owner changes after scheduler releases When a logical queue closes, its scheduler release is published once its work is idle. The firmware consumes the release against the queues that scheduler last ran. After the host has seen it consumed, the teardown parks the scheduler's render pair, and a later owner reinstalls a parked pair with fresh queue records. A successor on the same VM can find such a pair still installed after the release was published but before its consumption was seen. The pair is then reusable() and not foreign, so build_render_pair() rebinds it in place, rewriting the job-list head, pid and scheduler fields of queue records the firmware ran for the released scheduler, and publishes its first command, which carries the owner change. A GL client that recreates its context every few milliseconds next to three Vulkan clients slows the release enough to open this window. On the 7.2.10 line its successor published the first commands inside it and the GPU firmware crashed about 3 ms later without writing a crash log. At the crash the release had been consumed, the pairs had been rebound in place and their first commands for the new owner were still in flight. The 7.3 rebind path keeps the same reusable() test and has no release check. Order owner changes and scheduler releases: - A render pair whose context has an armed or unconsumed scheduler release is not rebound. The new owner retries until the teardown has parked the pair, then installs it with fresh records. The check is made when the rebind is chosen and again when it is committed. - A scheduler release waits while a render pair or compute queue that changed owner away from that context still has the new owner's first command in flight. A pair records at publication which scheduler its queues last ran for. A borrowed pair keeps a snapshot of that state. - The first command after an owner change waits while the scheduler the pair last ran for is being released. - The same holds for compute queues: a queue whose bound scheduler has an armed or unconsumed release is not rebound or re-armed for another owner, and the first command after a compute owner change (including a queue an idle live owner yielded through share_idle()) waits while the scheduler the queue was bound to is being released. - A pair parked without a consumed release keeps the scheduler it last ran for, so its next owner change still waits for that release. A park after the consumed release of the pair's scheduler clears it. - A publication refused because its pair is busy kicks the event worker on every poll, since the event worker observes the consumed release and parks the pair, and gives up after the job timeout instead of after 5000 polls. The normal path is unchanged. A pair whose owner does not change compares one word at publication. A release is delayed only while an owner change is in flight. After a context churn, a same-VM successor's first render waits until the previous context's release has been consumed and its pair parked, which successors on another VM already did. Measured on Neo on the 7.2.10 line with diagnostic switches that apply the first two rules: - four-client churn: 3 of 3 runs crashed within 10 to 120 s without them, 2 runs of 90 s passed with them - GLES: 15160 vs 15175 iterations, 5054 vs 5059 contexts per 90 s - Vulkan 4x MSAA render: 212643 vs 214050 iterations tools/asahi/g17-owner-release-order-tests.py compiles the release table, pair switch tracking and release readiness predicates against host stubs and checks the guard placement in the rebind and publication paths. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 15 + drivers/gpu/drm/asahi/g17/queue/render.rs | 25 ++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 15 + drivers/gpu/drm/asahi/g17/runtime.rs | 21 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 6 + .../gpu/drm/asahi/g17/runtime/render/grow.rs | 3 + .../drm/asahi/g17/runtime/render/observe.rs | 2 + .../drm/asahi/g17/runtime/render/prepare.rs | 14 + .../drm/asahi/g17/runtime/render/submit.rs | 36 ++- .../drm/asahi/g17/runtime/render/teardown.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- drivers/gpu/drm/asahi/g17/teardown.rs | 15 + tools/asahi/g17-owner-release-order-tests.py | 260 ++++++++++++++++++ 13 files changed, 414 insertions(+), 13 deletions(-) create mode 100755 tools/asahi/g17-owner-release-order-tests.py diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9a105bf9da03f..9ccdd703106b7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -589,6 +589,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index be19596e39fa6..392c8ee093757 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -172,6 +172,12 @@ pub(crate) struct Pair { /// A retired, never-executed group awaiting republication on this pair. replay: Option>, replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] @@ -261,6 +267,8 @@ impl Pair { replay: None, replay_since: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -798,6 +806,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -1596,6 +1609,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.is_some() || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 550a9ce5ed866..5b3f2a00bad6d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -31,6 +31,11 @@ pub(crate) struct ParkedPair { registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -91,6 +96,7 @@ impl Pair { registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -105,6 +111,13 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } @@ -230,6 +243,8 @@ impl ParkedPair { replay: None, replay_since: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4750d1f2e4438..67520641ab69a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -136,6 +136,7 @@ impl Registry { queue.share_idle()?; } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -144,6 +145,12 @@ impl Registry { .filter(|queue| queue.reusable()) { admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -174,6 +181,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -181,6 +189,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -1082,6 +1094,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 90ca9ed91c63a..135cd8240209b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -46,6 +46,9 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], closed: bool, pending_error: Option<(Arc, Error)>, @@ -163,6 +166,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -182,6 +186,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -341,6 +346,7 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], closed: false, pending_error: None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 856d6a0501548..8dd010428a3bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,6 +271,7 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; } if firmware.queues.render.grow.active != Some(request) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 9e59a854fe1fc..ef2da4515861d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 411b4f0fd962f..8403a0f506ce8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -137,6 +137,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -168,6 +175,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -467,6 +476,10 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { @@ -503,6 +516,7 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; (result, firmware.render_grow_ready()) }; drop(prepared); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index a27e2222955dd..982642adc43e8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -113,6 +113,20 @@ impl Backend { } } + /// A pair is borrowed, still settling, or waiting for the release of the + /// scheduler it last ran for to be consumed and the pair parked. The event + /// worker observes both, so kick it on every poll; time out only after the + /// same bound as a capacity wait. + fn wait_render_busy(&self, since: &mut Option) -> Result { + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + /// A closing owner may still hold its pair until exact retirement. Wait /// outside both the device mutex and preparation gate so the event worker /// and recovery can return capacity. No packet is firmware-visible here. @@ -140,7 +154,7 @@ impl Backend { packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); loop { @@ -185,11 +199,7 @@ impl Backend { } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since)?; continue; } result => result?, @@ -214,11 +224,7 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since)?; } } result => return result, @@ -394,6 +400,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0..eb04f80361955 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01..f8020e8962f84 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b..53518a76e91a4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py new file mode 100755 index 0000000000000..201f525718950 --- /dev/null +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the ordering between physical-queue owner changes and scheduler releases. + +The production release table, pair switch tracking, release readiness and compute +switch predicate are compiled against host stubs. This is not GPU execution. The +captured firmware crash (a render pair rebound in place and kicked by a same-VM +successor while its previous scheduler's release was published and the pair not +yet parked) must fail the same assertions once either rule is removed. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def ordered(source, *needles): + """Every needle occurs, in this order, inside `source`.""" + position = 0 + for needle in needles: + found = source.find(needle, position) + assert found >= 0, f'{needle!r} missing or out of order' + position = found + len(needle) + + +def check_guards(): + prepare = (ROOT / 'runtime/render/prepare.rs').read_text() + build = item(prepare, 'fn build_render_pair(') + # The in-place rebind is refused while the pair's scheduler release is pending. + ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + 'return Err(EBUSY);', 'Build::Rebind {') + rebind = item(prepare, 'fn rebind_pair(') + # A live donor that closed while the binding was prepared is caught at commit. + ordered(rebind, 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', + 'firmware.queues.teardown.displace(', 'pair.bind_owner(') + submit = (ROOT / 'runtime/render/submit.rs').read_text() + publish = item(submit, 'pub(super) fn publish_render(') + ordered(publish, '.owner_switch_from(&packet.context)', 'releasing_scheduler(scheduler)', + 'return Err(EBUSY);', 'acquire_owner(buffer_id)', 'pair.publish(') + render = (ROOT / 'queue/render.rs').read_text() + pair_publish = item(render, 'pub(crate) fn publish(') + # The switch is recorded only after the outer publication succeeded. + ordered(pair_publish, 'let outer_next = match result', 'self.ordinal = next_ordinal;', + 'self.switch = Some((self.executed, ordinal));', 'self.executed = scheduler;', + 'self.announce_publication(') + parked = (ROOT / 'queue/render/parked.rs').read_text() + # Parking keeps the last scheduler; only a consumed release clears it. + ordered(item(parked, 'pub(crate) fn park('), 'executed: self.executed,') + ordered(item(parked, 'pub(crate) fn commit('), 'executed: self.executed,', 'switch: None,') + detach = item((ROOT / 'runtime/render/teardown.rs').read_text(), 'fn detach_context(') + ordered(detach, '.park();', 'parked.release_consumed(context.scheduler_va());') + runtime = (ROOT / 'runtime.rs').read_text() + # A compute queue does not change owner while its scheduler is released. + ordered(item(runtime, 'pub(super) fn rebind_compute('), '.filter(|queue| queue.reusable())', + 'teardown.releasing(bound)', 'continue;', 'queue.bind_owner(owner, context.clone())?;') + ordered(item(runtime, 'pub(super) fn bind_rearmed('), 'teardown.releasing(bound)', 'return Err(EAGAIN);', + 'queue.bind_owner(') + ordered(item(runtime, 'fn publish_compute('), '.pending_switch_from()', 'queues.teardown.releasing(previous)', + 'return Err(EAGAIN);', 'queue.publish(') + # A busy pair (for example one whose scheduler release is still being consumed) + # kicks the event worker, which observes the release and parks the pair. + ordered(item(submit, 'fn wait_render_busy('), 'WORK_RING_WAIT_NS', 'return Err(ETIMEDOUT);', + 'self.shared.queue_events();', 'fsleep(') + for name, path in [('take_pair', 'runtime/render.rs'), ('take_render_observation', 'runtime/render/observe.rs')]: + body = item((ROOT / path).read_text(), f'fn {name}(') + ordered(body, 'entry.borrowed_switch = pair.switch_in_flight();', 'entry.pair.take()') + print('PASS guard placement: rebind decision, rebind commit, first publication, switch record, borrows, parked scheduler, compute owner changes, busy wait') + + +def fixture(): + teardown = (ROOT / 'teardown.rs').read_text() + assert ('#[derive(Clone, Copy, PartialEq, Eq)]\nenum Phase {\n Deferred,\n Draining,\n' + ' Released(u32),\n}') in teardown + pending = '\n'.join(item(teardown, name) for name in [ + 'pub(crate) fn displace(', 'pub(crate) fn close(', 'fn insert(', 'pub(crate) fn releasing(', + 'pub(crate) fn releasing_scheduler(', 'fn find(']).replace('pub(crate) ', '') + pending = pending.replace('self.entries.push(entry, GFP_KERNEL)?;', 'self.entries.push(entry);') + render = (ROOT / 'queue/render.rs').read_text() + switch_methods = '\n'.join(item(render, name) for name in [ + 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') + record = render[render.index(' let scheduler = packet.context.scheduler_va();'): + render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] + idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), + 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + compute_switch = item((ROOT / 'queue/compute.rs').read_text(), + 'pub(crate) fn switching_from(').replace('pub(crate) ', '') + compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') + return r''' +#![allow(dead_code)] +use std::sync::Arc; +type Error = i32; +type Result = std::result::Result; +struct Context { scheduler: u64, published: bool } +impl Context { + fn scheduler_va(&self) -> u64 { self.scheduler } + fn is_published(&self) -> bool { self.published } +} +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { Deferred, Draining, Released(u32) } +struct Entry { context: Arc, phase: Phase, armed: u64, announce_pending: bool } +struct Pending { entries: Vec } +impl Pending { +PENDING +} +struct Ticket { ordinal: u64 } +struct Active { ticket: Ticket } +struct Packet { context: Arc } +struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Option<()>, active: Vec, idle: bool } +impl Pair { +SWITCH + /// Host stand-in for a successful publication of `ordinal` (stand-in, not production). + fn publish(&mut self, packet: &Packet, ordinal: u64) { +RECORD + self.active.push(Active { ticket: Ticket { ordinal } }); + } + fn idle_for_context(&self, _: &Arc) -> Result { Ok(self.idle) } +} +struct RenderEntry { + context: Option>, + preparing: Option<()>, + growing: bool, + pair: Option>, + borrowed_switch: Option, +} +struct RenderState { entries: Vec> } +impl RenderState { +IDLE +} +struct Previous { context: Arc, publication: Option<(u64, u64)> } +struct Queue { previous: Option, bound: Option> } +impl Queue { +COMPUTE_SWITCH + fn idle_for_context(&self, context: &Arc) -> bool { + !self.bound.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } +} +struct ComputeEntry { queue: Option> } +struct Service { compute: Vec> } +impl Service { +COMPUTE_IDLE +} +fn context(scheduler: u64) -> Arc { Arc::new(Context { scheduler, published: true }) } +fn main() { + let (b, c, d) = (context(0x1000), context(0x2000), context(0x3000)); + // Captured crash: GL context B closes, its release is published, then the same-VM + // successor C finds B's pairs still installed (not parked) and rebinds them in place. + let mut table = Pending { entries: Vec::new() }; + table.close(&b, 1).unwrap(); + assert!(table.releasing(&b), "armed release refuses an owner change"); + table.entries[0].phase = Phase::Released(67); + assert!(table.releasing(&b), "published, unconsumed release refuses an owner change"); + assert!(table.releasing_scheduler(0x1000) && !table.releasing_scheduler(0x2000)); + // Consumption and detach remove the entry; the pair is parked and reinstalled fresh. + table.entries.clear(); + assert!(!table.releasing(&b) && !table.releasing_scheduler(0x1000)); + // A displacement from a live owner is not a release; its later close arms one. + table.displace(&d, false, 2).unwrap(); + assert!(!table.releasing(&d), "a live displaced owner may still lose an idle pair"); + table.close(&d, 3).unwrap(); + assert!(table.releasing(&d), "the donor's close arms its release"); + // An unpublished context has no firmware scheduler state and no release. + let quiet = Arc::new(Context { scheduler: 0x4000, published: false }); + table.close(&quiet, 4).unwrap(); + assert!(!table.releasing(&quiet)); + + // Switch tracking across an owner change of one pair. + let mut pair = Pair { executed: 0, switch: None, replay: None, active: Vec::new(), idle: true }; + assert_eq!(pair.owner_switch_from(&c), None, "a fresh graph has no previous scheduler"); + pair.publish(&Packet { context: d.clone() }, 0); + pair.active.clear(); + assert_eq!(pair.owner_switch_from(&d), None, "the same owner never switches"); + assert_eq!(pair.owner_switch_from(&c), Some(0x3000)); + pair.publish(&Packet { context: c.clone() }, 1); + assert_eq!(pair.switch_in_flight(), Some(0x3000)); + let mut render = RenderState { entries: vec![Some(RenderEntry { + context: Some(c.clone()), preparing: None, growing: false, pair: Some(Box::new(pair)), borrowed_switch: None, + })] }; + assert!(!render.teardown_idle(&d).unwrap(), "release waits for the new owner's first command"); + assert!(render.teardown_idle(&b).unwrap(), "unrelated releases are not held"); + // A borrowed pair is judged by its snapshot; a stale snapshot outside a borrow is ignored. + let borrowed = render.entries[0].as_mut().unwrap(); + let pair = borrowed.pair.take().unwrap(); + borrowed.borrowed_switch = pair.switch_in_flight(); + borrowed.preparing = Some(()); + assert!(!render.teardown_idle(&d).unwrap(), "a borrowed switching pair still holds the release"); + let entry = render.entries[0].as_mut().unwrap(); + entry.preparing = None; + entry.pair = Some(pair); + entry.pair.as_mut().unwrap().active.clear(); + assert!(render.teardown_idle(&d).unwrap(), "retirement of the first command releases the hold"); + let entry = render.entries[0].as_mut().unwrap(); + entry.pair.as_mut().unwrap().active.push(Active { ticket: Ticket { ordinal: 1 } }); + entry.pair = None; + assert!(render.teardown_idle(&d).unwrap(), "a stale snapshot without a borrow holds nothing"); + + // Compute: only a published first command after the change holds the release. + let mut service = Service { compute: vec![Some(ComputeEntry { queue: Some(Box::new(Queue { + previous: Some(Previous { context: b.clone(), publication: None }), bound: Some(c.clone()), + })) })] }; + assert!(service.compute_idle(&b), "an unpublished rebind does not hold the release"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous.as_mut().unwrap().publication = Some((1, 0)); + assert!(!service.compute_idle(&b), "compute release waits for the new owner's first command"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous = None; + assert!(service.compute_idle(&b)); + assert!(!service.compute_idle(&c), "the bound owner still needs its own retirement"); + println!("PASS release table gate, pair switch hold, borrowed snapshot and compute switch hold"); +} +'''.replace('PENDING', pending).replace('SWITCH', switch_methods, 1).replace('RECORD', record) \ + .replace('IDLE', idle, 1).replace('COMPUTE_SWITCH', compute_switch).replace('COMPUTE_IDLE', compute_idle) + + +def main(): + check_guards() + code = fixture() + mutations = { + 'fixed': (None, None, None), + 'no-release-gate': ('.is_some_and(|index| self.entries[index].phase != Phase::Deferred)', + '.is_some_and(|_| false)', 'armed release refuses an owner change'), + 'no-render-hold': ('Some(pair) => pair.switch_in_flight() == scheduler,', 'Some(_) => false,', + "release waits for the new owner's first command"), + 'no-compute-hold': ('queue.idle_for_context(context) && !queue.switching_from(context)', + 'queue.idle_for_context(context)', + "compute release waits for the new owner's first command"), + } + with tempfile.TemporaryDirectory(prefix='g17-owner-release-') as directory: + root = Path(directory) + for name, (old, new, expected) in mutations.items(): + current = code + if old is not None: + assert current.count(old) == 1, name + current = current.replace(old, new) + path = root / (name + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], + check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if old is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() From 1d478ee6c06b56b48782d08b693d9aa9e5c43435 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:19:47 -0400 Subject: [PATCH 4/9] drm/asahi: keep G17 render kick ring storage for the life of the device A pair installs the client alias of each kick ring into its VM when the pair is installed (append_driver_mappings), and the VM keeps that alias until it closes: a client that gets the pair back later reuses it. Under identity pressure, reclaim_unused_render_spare() takes an installed pair whose graph never started, for compute or for another render owner's queue IDs, and drops it together with its rings. That frees the rings' shared lower addresses while the VM that installed the pair, which may outlive the closed queue (one DRM file with several queues), still aliases the old pages there. A later ring placed at the same address and handed to that VM cannot be aliased: prepare_client_alias() finds the VM's mapping naming other pages and fails with EFAULT, and the client's render is reported as a lost device. On the 7.2.10 line held GLES clients hit this every few seconds while fresh clients came and went. Keep the storage of a dropped ring and reuse it for later rings. An address then keeps naming the same pages, so every alias a client keeps stays valid; a reused ring is zeroed and keeps an existing alias. Storage of a ring that ever wrote an entry is retained and never reused. A new ring is counted only once its storage exists, reserving room to return it then, so a dropped ring never allocates and a failed creation leaves no reservation behind. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/kick.rs | 163 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/render.rs | 8 +- .../drm/asahi/g17/runtime/render/prepare.rs | 3 + 4 files changed, 161 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index a2f8094dfb996..7aaca95992f8d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -291,10 +298,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -330,6 +421,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -364,14 +458,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -399,14 +524,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -416,19 +553,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -441,7 +578,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9129418f4a16f..04dd027a975f7 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -480,6 +480,8 @@ struct Firmware { registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -981,6 +983,7 @@ impl Gpu { registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 392c8ee093757..b69cd4d565736 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -188,6 +188,7 @@ impl Pair { pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -225,8 +226,9 @@ impl Pair { } let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -240,7 +242,7 @@ impl Pair { word, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 8403a0f506ce8..5795c14dbc1d6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,6 +29,7 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), @@ -255,6 +257,7 @@ impl super::super::Backend { Pair::new( alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, From 7304ddbf6d4d47b7976efcd20623524c9c53b0b1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:21:17 -0400 Subject: [PATCH 5/9] drm/asahi: keep G17 render capacity waits from stopping the runtime A render packet that waits in publication for graph capacity already holds its admission lease, and the entry of a pair under construction has no installed pair. Admission, the capacity waits and the busy-pair polls each have their own budget, so a publication can outlive the two-second scheduler watchdog. timed_out() then counts the expiry towards the timeout storm limit, and timeout_render() looks the pair up for an unpublished packet, fails with EIO, and the handler loses the device for every client. Eight such expiries within 30 seconds, which a burst of new clients reaches, lose the device permanently even without the lookup. On the 7.2.10 line a burst of new clients whose render free lists did not fit the shared lower GPU VA window stopped the runtime two seconds after the first wait began. Each retry of such a build also populates the full free-list backing, tens of MiB with ray-query scratch reservations, before the window is searched, and frees it again after the refusal. With the 1 ms capacity poll, sixteen concurrent builders reached several GiB of transient kernel allocations. - A watchdog expiry for a packet that is not published returns NoHang without counting towards the storm limit and keeps the packet's place in the backlog: its publisher owns the bounded wait and the result. - If the timeout collector cannot be allocated, the expiry is retried at the next watchdog period instead of stopping the runtime. - A refused free-list placement is not attempted again for the same or a larger size until free-list backing is freed, apart from one probe per 100 ms across all callers, so waiting builders allocate nothing. - While the window refuses free-list backing, idle retained compute pools are given back regardless of the warm cap. The host resource tests check the placement gate and the watchdog guard placement. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/freelist.rs | 91 ++++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 9 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 9 ++ .../drm/asahi/g17/runtime/compute_control.rs | 5 +- .../drm/asahi/g17/runtime/render/settle.rs | 6 +- tools/asahi/g17-resource-tests.py | 63 +++++++++++++ 6 files changed, 167 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 49544300531a8..3b31c3cead064 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -69,19 +69,81 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; Self::from_backing(backing, blocks, layout) } fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { @@ -147,13 +209,16 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let (backing, [global, client]) = alloc.lower_with_aliases( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - [render_global, client], - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 67520641ab69a..6d5b589b5c4af 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1499,9 +1499,16 @@ impl super::Firmware { if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 822e1d9d07713..0d77ef4a54476 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -190,6 +190,15 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34ae..80f88d09191b4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -50,7 +50,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index b63c3dc5260ef..5db82a08bc8d1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 36d4ced3510bb..35f970799b086 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -232,6 +232,67 @@ def item(s,start): subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) subprocess.run([str(root/'unused-render-guard-controls')],check=True) +def lower_window_gate_tests(root, source): + s=(source/'g17/freelist.rs').read_text() + start=s.index('static LOWER_FREED');end=s.index('pub(crate) struct Pages {') + gate=s[start:end] + fixture=root/'lower-window-gate-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, Ordering}; +type Result = std::result::Result; +const ENOSPC:i32=-28;const ENOMEM:i32=-12; +static CLOCK: AtomicU64 = AtomicU64::new(1_000_000_000); +fn now_ns()->u64 {CLOCK.load(Ordering::Relaxed)} +mod freelist { +use super::*; +'''+gate+''' +pub(crate) fn freed() {LOWER_FREED.fetch_add(1, Ordering::AcqRel);} +pub(crate) fn place(size: usize, f: impl FnOnce() -> Result) -> Result {place_lower(size, f)} +} +use freelist::*; +fn attempt(size:usize, result:Result, calls:&mut u32)->Result {place(size,||{*calls+=1;result})} +fn main(){ + let mut calls=0; + assert_eq!(attempt(100,Ok(1),&mut calls),Ok(1));assert_eq!(calls,1);assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,2);assert!(lower_pressure()); + for _ in 0..1000 {assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));} + assert_eq!(attempt(200,Ok(1),&mut calls),Err(ENOSPC)); + assert_eq!(calls,2,"a refused size is not placed again while nothing was freed"); + assert_eq!(attempt(50,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"a smaller size still probes"); + assert_eq!(attempt(60,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"the smaller refusal now gates larger sizes"); + freed();assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,4,"freed backing lets a waiter retry"); + // One probe per interval across all callers, then the gate closes again. + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Ok(7),&mut calls),Ok(7));assert_eq!(calls,6);assert!(!lower_pressure(),"a fitting placement clears pressure"); + assert_eq!(attempt(100,Err(ENOMEM),&mut calls),Err(ENOMEM));assert!(!lower_pressure(),"only window refusals gate"); + println!("lower window: refused free-list placements are not repeated until backing is freed or one probe per 100 ms"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'lower-window-gate-controls')],check=True) + subprocess.run([str(root/'lower-window-gate-controls')],check=True) + +def watchdog_publication_guard_tests(source): + # A packet still in publication (capacity or graph waits) must never reach + # pair lookups, the timeout storm count or a collector allocation from its + # watchdog: each of them can stop the runtime for every client. + text = (source / 'g17/runtime/backend.rs').read_text() + at = text.index(' fn timed_out(') + body = text[at:text.index('\n }\n', at)] + guard = body.index('if !packet.is_published() {\n return sched::Status::NoHang;') + assert guard < body.index('self.release_backlog(&packet);') < body.index('self.shared.watchdog_timeout()'), 'timed_out' + for path, name in [('g17/runtime/render/settle.rs', 'pub(in crate::g17) fn timeout_render('), + ('g17/runtime/compute_control.rs', 'pub(super) fn timeout_compute(')]: + text = (source / path).read_text() + at = text.index(name) + body = text[at:text.index('\n }\n', at)] + assert 'let Ok(mut deferred) = DeferredBatch::worker() else {' in body, name + assert 'DeferredBatch::worker()?' not in body, name + print('watchdog: unpublished packets and collector allocation failure never stop the runtime') + def shared_alias_tests(root, source): s=(source/'g17/object.rs').read_text() at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 @@ -380,4 +441,6 @@ def shared_alias_tests(root, source): with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: admission_tests(Path(temporary), DRIVER) queue_tests(Path(temporary), DRIVER) + lower_window_gate_tests(Path(temporary), DRIVER) + watchdog_publication_guard_tests(DRIVER) shared_alias_tests(Path(temporary), DRIVER) From 72d57c0b9ac9b8d5fcd58c59eebfa7f9c605399c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:23:10 -0400 Subject: [PATCH 6/9] drm/asahi: wait for G17 contexts of closing queues at queue creation Every logical queue leases one of the 58 application contexts of the TTBAT (5..62) when it is created. A closed queue keeps that lease until its scheduler release has been consumed and the reclaim worker returns it; retained compute queues and render pairs keep only storage, not the context. After heavy device churn, contexts of queues closed moments earlier can therefore still be leased while fewer than 58 queues are open, and QUEUE_CREATE fails with ENOSPC from new_execution_context() although the logical-queue count admitted the queue. Userspace creates its kernel queue at the first submission and reports that as a lost device. When the context allocation is refused and some leased contexts belong to closed queues, kick the event worker and retry for a bounded time with neither the device mutex nor the preparation lease held. Only open queues then count against the limit. Nothing of the new queue is visible while it waits, so a creator with a pending signal returns ERESTARTSYS. A refusal that remains is counted and named as an exhausted execution context, next to the existing logical-queue refusal, in the capacity diagnostics. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/capacity.rs | 5 +- drivers/gpu/drm/asahi/g17/queue/admission.rs | 4 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 52 +++++++++++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index db2561d4d062f..bcbbb2e7e875d 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -24,9 +24,10 @@ pub(super) enum Site { LowerVa, UpperVa, AliasVa, + ExecutionContexts, } -const SITE_COUNT: usize = Site::AliasVa as usize + 1; +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; /// First and power-of-two refusals per site keep a pressure storm bounded in @@ -78,7 +79,7 @@ impl kernel::debugfs::Writer for View { Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, - Site::LowerVa, Site::UpperVa, Site::AliasVa, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, ].iter().zip(REFUSALS.iter()) { writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 0165a00568118..7fc4d79a5f95c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -372,6 +372,10 @@ impl Pool { GFP_KERNEL, ) } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6d5b589b5c4af..ff1214f655ae8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -443,24 +443,54 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, From a4351b50cd8d6f194bb71252acf9c9e2cf636db4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:25:34 -0400 Subject: [PATCH 7/9] drm/asahi: keep G17 publication waits out of the submit ioctl Dependency-free renders are published from the submit ioctl, which holds the logical queue's mutex. try_render_early() only takes an admission lease that is free at once, but render_with_lease() can then still sleep for contention: render identities and capacity (up to a job timeout), a borrowed or settling pair (up to a job timeout), work states (up to 1000 polls), recovery settling, and an uninterruptible wait on a closed preparation gate. A client killed while its ioctl waited stalled for that long, and on the 7.2.10 line, where the budgets were longer, for about 30 s. The submit ioctl no longer sleeps for contention. An ioctl publication that would wait returns its admission lease and its publication claim before anything became firmware-visible, and leaves the command to the scheduler like a command with dependencies; later commands of the queue then wait behind it in the backlog. The scheduler's worker keeps the existing bounded waits. Uncontended ioctl publication is unchanged. Compute publication also failed the packet with EBUSY when the context had no free work state, although work states free as the context's earlier commands retire. Wait for one under the packet's admission deadline instead, as render publication does. tools/asahi/g17-compute-deadline-tests.py runs the production compute publication with a context that has no free work state and checks that it waits until the admission deadline. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 6 ++ drivers/gpu/drm/asahi/g17/job.rs | 7 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 18 +++- .../drm/asahi/g17/runtime/render/prepare.rs | 21 +++-- .../drm/asahi/g17/runtime/render/submit.rs | 82 ++++++++++++++++--- tools/asahi/g17-compute-deadline-tests.py | 13 ++- 6 files changed, 124 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 13fb35d03d04d..e43ee32aa614c 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -405,6 +405,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 0588539c1ff75..575a9a7adc976 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -196,6 +196,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ff1214f655ae8..a9622d5909499 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1312,6 +1312,7 @@ impl Backend { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; + let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1328,7 +1329,22 @@ impl Backend { Backend::ensure_compute(self, scratch.pages(), &admission)?; admission.check()?; let lease = self.shared.preparations.enter_until(&admission)?; - packet.completion.work_state()?.acquire()?; + // Work states free as this context's earlier commands retire. + // Wait for one under the packet's admission deadline, as render + // publication does, instead of failing the packet. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + admission.check()?; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 5795c14dbc1d6..4c643c46eebad 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -78,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -577,8 +583,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -593,8 +599,8 @@ impl super::super::Backend { // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling.slot(), packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -606,8 +612,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 982642adc43e8..b77fc13b7ed2f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,29 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -30,6 +52,7 @@ impl Backend { &self, packet: &Arc, mut next: Option, + sleep: bool, ) -> Result { let pool = { let state = self.shared.state.lock(); @@ -49,6 +72,10 @@ impl Backend { let lease = if let Some(lease) = next.take() { lease } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { // Atomics only: the callback runs under the pool mutex. if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { @@ -70,7 +97,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started)?; + self.wait_render_capacity(started, sleep)?; } drop(reclaimed); continue; @@ -104,7 +131,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - self.wait_render_capacity(started)?; + self.wait_render_capacity(started, sleep)?; } drop(reclaimed); } @@ -117,7 +144,10 @@ impl Backend { /// scheduler it last ran for to be consumed and the pair parked. The event /// worker observes both, so kick it on every poll; time out only after the /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option) -> Result { + fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } let now = super::super::now_ns(); if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { return Err(ETIMEDOUT); @@ -130,7 +160,10 @@ impl Backend { /// A closing owner may still hold its pair until exact retirement. Wait /// outside both the device mutex and preparation gate so the event worker /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64) -> Result { + fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } self.wait_capacity(started) } @@ -149,7 +182,10 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early)?; + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; @@ -169,7 +205,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -180,6 +220,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { + if !sleep { + return Err(FALLBACK); + } if state_polls == 1000 { return Err(ETIMEDOUT); } @@ -192,21 +235,21 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, sleep) { Err(error) if error == ENOSPC => { - self.wait_capacity(capacity_started)?; + self.wait_render_capacity(capacity_started, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since)?; + self.wait_render_busy(&mut busy_since, sleep)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_capacity(capacity_started)?; + self.wait_render_capacity(capacity_started, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { @@ -217,6 +260,9 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { + if !sleep { + return Err(FALLBACK); + } if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { return Err(ETIMEDOUT); } @@ -224,7 +270,9 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy_since)?; + self.wait_render_busy(&mut busy_since, sleep)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -267,6 +315,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py index b551cbaa96339..eca8b73c5b12c 100644 --- a/tools/asahi/g17-compute-deadline-tests.py +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -52,8 +52,8 @@ def fixture(): fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);} mod kernel{pub mod time{ pub fn msecs_to_jiffies(ms:u32)->u32{ms} - pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms)}} - pub mod delay{pub fn fsleep(d:super::Delta){crate::advance(d.0*1_000_000);}} + pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms*1_000_000)}pub fn from_micros(us:i64)->Self{Self(us as u64*1_000)}} + pub mod delay{pub fn fsleep(d:super::Delta){crate::advance(d.0);}} }} ''' + admission + r''' struct Wait; @@ -70,7 +70,7 @@ def fixture(): struct Scratch;impl Scratch{fn pages(&self)->u16{0}} enum Validated{Compute{scratch:Scratch}} struct Status;impl Status{fn get(&self)->i32{if now_ns()>=STATUS.load(Ordering::SeqCst){-5}else{0}}} -struct Work;impl Work{fn acquire(&self)->Result{Ok(())}} +struct Work;impl Work{fn acquire(&self)->Result{if MODE.load(Ordering::SeqCst)==4{Err(EBUSY)}else{Ok(())}}} struct Completion;impl Completion{fn status(&self)->Status{Status}fn work_state(&self)->Result{Ok(Work)}} struct Packet{command:Validated,completion:Completion} impl Packet{fn check_dependencies(&self)->Result{Ok(())}fn cancel_requested(&self)->bool{now_ns()>=CANCEL.load(Ordering::SeqCst)}fn is_published(&self)->bool{false}} @@ -109,6 +109,13 @@ def fixture(): if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),4);assert_eq!(now_ns(),WORK_RING_WAIT_NS);} println!("PASS mode={} result={:?} calls={} elapsed_ms={}",mode,r,CALLS.load(Ordering::SeqCst),now_ns()/1_000_000); } + // A context without a free work state waits for one under the admission + // deadline instead of failing the packet at once. + MODE.store(4,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); + let r=b.compute_packet(&p,&mut DeferredBatch); + assert_eq!(r,Err(EBUSY),"a busy work state ends at the admission deadline"); + assert!(now_ns()>=WORK_RING_WAIT_NS&&now_ns()<=WORK_RING_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); + println!("PASS busy work state waits elapsed_ms={}",now_ns()/1_000_000); for status in [false,true] { MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); if status{STATUS.store(10_000_000,Ordering::SeqCst);}else{CANCEL.store(10_000_000,Ordering::SeqCst);} From c556dacd085b148da86b47792dc34d14f02ace86 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:26:04 -0400 Subject: [PATCH 8/9] drm/asahi: rerun the G17 event pass for completions missed while a pair is borrowed Event passes observe only render pairs that are installed in their entry. A pair that preparation, growth or an observation has borrowed is skipped, and nothing requested another pass when it came back: the completion of work published on that pair waited for an unrelated later firmware event, or for the one-millisecond rescan when no further event arrived. Record a completion event that arrives for a borrowed pair and have the borrower request an event pass when it returns the pair. Growth needs no request of its own; its work item always queues one when it finishes. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 24 ++++++++++++++++++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 2 ++ .../drm/asahi/g17/runtime/render/observe.rs | 2 +- .../drm/asahi/g17/runtime/render/prepare.rs | 9 +++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 135cd8240209b..88340442bbcae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -50,6 +50,9 @@ struct Entry { /// nothing, so the snapshot can only be stale towards an older switch. borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -133,6 +136,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -348,6 +359,7 @@ impl Registry { borrowed_dependencies: None, borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -878,9 +890,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 8dd010428a3bb..e3cb174721aaa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -273,6 +273,8 @@ impl crate::g17::Shared { entry.borrowed_dependencies = None; entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index ef2da4515861d..4ede20d316c64 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -165,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 4c643c46eebad..2acd464bbeea4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -419,6 +419,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -526,6 +529,9 @@ impl super::super::Backend { entry.pair = Some(pair); entry.borrowed_dependencies = None; entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -690,6 +696,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues From 8744e9417249175511480765dbcc99f0a09c0fb9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 22 Oct 2026 19:26:05 -0400 Subject: [PATCH 9/9] drm/asahi: reuse G17 compute timestamp pages within a context Every compute command got a fresh page for its firmware start/end words: a GEM object, its write-combined kernel mapping and a GPU mapping were set up in the submit ioctl, and torn down again when the command's completion was dropped. On Neo a lone dispatch spends ~95 us from its fence signal to the next command's ioctl, against ~17 us for a render, which has no such per-command allocation. Keep up to eight retired pages per context, mapped in its VM, and hand them out again with both words cleared. A page returns only when its completion is dropped, which follows exact retirement or a kill witness, so the firmware no longer writes it. Commands beyond the retained count still allocate and free their page. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 88 +++++++++++++++++++++++-- drivers/gpu/drm/asahi/g17/context.rs | 6 ++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index e43ee32aa614c..ea58cfb8de286 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,81 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +249,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -220,7 +296,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -282,14 +358,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -339,7 +415,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index fe874992cffeb..c93ab25d55ce3 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -194,12 +196,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status }