From 7fcc7fd38c43b6aa64ea2270dcd32cdd57d19f37 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 04:16:02 +1100 Subject: [PATCH 01/11] drm/asahi: carry G17 admission prerequisites to 7.3 Carry three of Dj's commits from omacom/linux-aurora#151 that the G17 capacity work depends on: "reclaim unused G17 render warming under queue pressure", "recover unused initial G17 queue IDs for compute" and "find common addresses for G17 render pool backing". Keep the 7.3 memory accounting implementation when splitting the common render-pool backing constructor. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/freelist.rs | 21 +- drivers/gpu/drm/asahi/g17/kick.rs | 72 +++- drivers/gpu/drm/asahi/g17/object.rs | 77 ++++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 129 ++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 14 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 12 +- drivers/gpu/drm/asahi/g17/runtime.rs | 18 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 121 +++++- .../drm/asahi/g17/runtime/render/prepare.rs | 4 +- .../drm/asahi/g17/runtime/render/submit.rs | 88 ++++- drivers/gpu/drm/asahi/mmu.rs | 33 +- tools/asahi/g17-resource-tests.py | 357 ++++++++++++++++++ 12 files changed, 892 insertions(+), 54 deletions(-) create mode 100644 tools/asahi/g17-resource-tests.py diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc..6e677e6f761b6 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -76,12 +76,15 @@ pub(crate) struct Pages { } impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( + let backing = alloc.lower( TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -144,14 +147,14 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let (backing, [global, client]) = alloc.lower_with_aliases( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + )?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86d..e2a9bb643b892 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -17,7 +17,7 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. +/// Minimum render-channel capacity protected from compute-only allocation. const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a @@ -47,6 +47,7 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + initial_render: [Option; 2], compute_publication: u64, leased: usize, } @@ -55,11 +56,29 @@ impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + initial_render: [None; 2], compute_publication: 0, leased: 0, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -83,15 +102,45 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + let initial = if engine == DataMaster::Compute && self.leased == QID_COUNT { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| { + matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) + }) + .count(); + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let replacement = usize::from(initial.is_some()); + if QID_COUNT - self.leased + replacement + <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) + { + return Err(ENOSPC); + } } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) .ok_or(ENOSPC)?; let id = Id { qid: index as u8, @@ -102,7 +151,11 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } @@ -127,6 +180,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae..a4f06a999eddc 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -161,6 +161,17 @@ impl KernelObject { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) } + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) + } + /// Maps an exact backing subrange, retaining an explicit unmapped guard. pub(crate) fn map_range( &self, @@ -341,6 +352,72 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + )? + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + )? + }; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + )?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f..6e99160729034 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -115,18 +115,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,8 +177,51 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; + } + } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -188,9 +233,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -420,8 +487,33 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -468,6 +560,17 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index bf64688841105..22e3fb49b940f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -152,6 +152,7 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, descriptor_flags: [u32; 2], installed: [bool; 2], @@ -243,6 +244,7 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, descriptor_flags, installed: [false; 2], @@ -507,9 +509,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -734,9 +744,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -1742,6 +1749,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 1025b3b01514d..034da4dd2eeea 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -48,11 +48,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -194,6 +203,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9..40d2a54b59e63 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -74,9 +74,7 @@ impl Registry { ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), @@ -624,7 +622,19 @@ impl Backend { Rebind::Grow => { drop(state); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf2045526..30da1b09eedfa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -55,6 +55,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -198,6 +204,113 @@ impl RenderState { } impl Registry { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); @@ -262,12 +375,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index ff2f34ae14cda..13e0e9a9d6d40 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -572,9 +572,11 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { + match self.build_render_pair(sibling.slot(), packet, true) { Err(error) if error == EBUSY || error == EAGAIN => {} result => result?, } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c..abea0b4c06f29 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -23,6 +23,83 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + match pool.acquire_render(self.owner, Request::of(packet)) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + return Err(error); + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + fn render_with_lease( &self, packet: &Arc, @@ -37,16 +114,7 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + let admission = self.acquire_render_with_ids(packet, early)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 5cad3d096a5c8..4322c88de0651 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1886,6 +1886,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1934,7 +1965,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py new file mode 100644 index 0000000000000..73d5fa52b9833 --- /dev/null +++ b/tools/asahi/g17-resource-tests.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual G17 admission, queue-ID, and unused-graph eligibility methods. + +Run with Python and rustc; temporary host binaries are removed after the checks. +""" +from pathlib import Path +import os +import re +import subprocess +import tempfile + +DRIVER = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi' + +def admission_tests(root, source): + s=(source/'g17/queue/admission.rs').read_text();cfg=(source/'hw/t8140.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + mmu=(source/'mmu.rs').read_text() + limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) + header='''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n'''+constants+'}\n'+limits+'\n' + methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + fn main(){ + let mut state=State::new(); + for owner in 1..=32 { + assert!(state.refusal(owner).is_ok()); + let slot=state.acquire(owner).expect("existing owner fits"); + state.release(slot); + } + assert_eq!(state.active,0,"all submitted work retired"); + let retained=state.owners.iter().filter(|o|**o!=0).count(); + assert_eq!(retained,63); + assert_eq!(state.refusal(33),Err(ENOSPC)); + assert!(state.acquire(33).is_none()); + println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); + // Yield one unused spare for each additional logical owner. The runtime + // separately proves that these physical graphs have never been published. + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); + assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); + println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); + let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; + guarded.take_idle(spare,1); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); + guarded.release(spare as u8); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(3),Some(2)),"owner mismatch"); + guarded.quarantined |= 1u64< = std::result::Result; + const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} + #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} + #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} + '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name+'(')+'\n' for name in ['reserve_initial_render','reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' + test=''' + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} + fn main(){ + for render in [0,2,16,32,112,126] { + let mut q=fresh(); + for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} + let mut compute=0; + while q.reserve(1000+compute,DataMaster::Compute,None).is_ok(){compute+=1;} + assert_eq!(compute as usize,QID_COUNT-render.max(COMPUTE_RENDER_RESERVE)); + } + let mut q=fresh();let first=q.reserve(1,DataMaster::Tiling,None).unwrap(); + assert!(q.can_cancel_unpublished(first));q.publish(first).unwrap(); + assert!(!q.can_cancel_unpublished(first));assert_eq!(q.cancel_unpublished(first),Err(EINVAL));assert_eq!(q.leased,1); + let unused=q.reserve(2,DataMaster::Fragment,None).unwrap();q.cancel_unpublished(unused).unwrap();assert_eq!(q.leased,1); + let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); + assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); + println!("queue IDs: six render/compute capacity boundaries; compute-only render reserve retained; published and stale identities protected"); + // Reproduce the hardware snapshot: 124 installed render identities, two + // installed compute identities, and two initial reservations fill128 IDs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + assert_eq!(q.reserve_initial_render(),Err(EINVAL)); + let initial=q.initial_render; + for owner in 1000..1002 {let id=q.reserve(owner,DataMaster::Compute,Some(4)).unwrap();q.publish(id).unwrap();} + let mut installed=Vec::new(); + for owner in 3..127 {let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();q.publish(id).unwrap();installed.push(id);} + assert_eq!(q.leased,QID_COUNT); + for (n,old) in initial.into_iter().enumerate() { + let old=old.unwrap();let id=q.reserve(2000+n as u64,DataMaster::Compute,None).unwrap(); + assert_eq!(id.qid,old.qid);assert_eq!(q.leased,QID_COUNT); + assert!(!q.can_cancel_unpublished(old));assert_eq!(q.cancel_unpublished(old),Err(EINVAL)); + q.publish(id).unwrap(); + } + assert_eq!(q.reserve(3000,DataMaster::Compute,None),Err(ENOSPC)); + for id in installed {assert!(!q.can_cancel_unpublished(id));assert_eq!(q.entries[id.qid as usize].unwrap().id,id);} + // A bootstrap token that became published is never replaced. + let mut q=fresh();q.reserve_initial_render().unwrap();let protected=q.initial_render[0].unwrap();q.publish(protected).unwrap(); + for owner in 3..129 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + assert_eq!(q.leased,QID_COUNT); + assert_eq!(q.reserve(1000,DataMaster::Compute,None).unwrap().qid,1); + assert_eq!(q.reserve(1001,DataMaster::Compute,None),Err(ENOSPC)); + assert_eq!(q.entries[0].unwrap().id,protected); + // Replacement must not consume the protected render minimum. + let mut q=fresh();q.reserve_initial_render().unwrap(); + for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 1000..1112 {q.reserve(owner,DataMaster::Compute,None).unwrap();} + assert_eq!(q.leased,QID_COUNT);assert_eq!(q.reserve(2000,DataMaster::Compute,None),Err(ENOSPC)); + assert!(q.initial_render.iter().all(Option::is_some)); + // Ordinary free capacity leaves the initialization reservations untouched. + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + let id=q.reserve(1000,DataMaster::Compute,Some(4)).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); + println!("initial reservations: full-namespace compute admission restored; physical count unchanged; published and stale identities plus render minimum protected"); + + } + ''' + fixture=root/'queue-id-capacity-controls.rs';fixture.write_text(header+test) + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'queue-id-capacity-controls')],check=True) + subprocess.run([str(root/'queue-id-capacity-controls')],check=True) + s=(source/'queue/render/parked.rs').read_text();guard=item(s,'pub(crate) fn can_yield_unused') + fixture=root/'unused-render-guard-controls.rs';fixture.write_text('''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + struct Context; + struct Pair {owner:Option,ordinal:u64,registered:Option,outer_started:u8,previous:Option<()>,graph_started:bool,backing:Result} + impl Pair { + fn context(&self)->&Context {&Context} + fn can_park_backing(&self,_:&Context)->Result{self.backing} + '''+guard+''' + } + fn fresh()->Pair {Pair {owner:Some(1),ordinal:0,registered:None,outer_started:0,previous:None,graph_started:false,backing:Ok(true)}} + fn main(){ + assert_eq!(fresh().can_yield_unused(),Ok(true)); + let mut p=fresh();p.owner=None;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.ordinal=1;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.registered=Some(2);assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.outer_started=4;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.graph_started=true;assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Ok(false);assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Err(-5);assert_eq!(p.can_yield_unused(),Err(-5)); + println!("unused graph: eight eligibility/error controls protect every publication boundary"); + } + ''') + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) + subprocess.run([str(root/'unused-render-guard-controls')],check=True) + +def shared_alias_tests(root, source): + s=(source/'g17/object.rs').read_text() + at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + method=s[at:end] + fixture=root/'shared-alias-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::{cell::RefCell,rc::Rc,ops::Range}; +type Result = std::result::Result;type Error=i32; +const EINVAL:i32=-22;const EOVERFLOW:i32=-75;const ENOSPC:i32=-28;const EIO:i32=-5; +mod hw {pub mod t8140 {pub mod dynamic {pub const LOWER:std::ops::Range=0..256;}}} +mod mmu {pub const UAT_PGSZ:usize=16;pub type Prot=u8;pub type Vm=super::Vm;pub type KernelMapping=super::Mapping;} +#[derive(Clone,Copy)] enum CpuMap {WriteCombined} +#[derive(Clone)] struct Region {vm:usize,start:u64,size:u64,object:u64} +#[derive(Default)] struct State {regions:Vec,allocations:u64,attempts:usize,fault:Option} +struct Vm {id:usize,state:Rc>} +struct Uat {lower:Vm} +impl Uat {fn kernel_lower_vm(&self)->&Vm {&self.lower}} +struct Device {state:Rc>} +struct Mapping {vm:usize,address:u64,size:u64,object:u64,state:Rc>} +impl Mapping {fn iova(&self)->u64 {self.address}} +impl Drop for Mapping {fn drop(&mut self) { + let mut state=self.state.borrow_mut(); + if self.vm==0 {assert!(!state.regions.iter().any(|r|r.vm!=0&&r.object==self.object&&r.start==self.address),"aliases must retire before canonical reservation");} + let index=state.regions.iter().position(|r|r.vm==self.vm&&r.start==self.address&&r.object==self.object).unwrap(); + state.regions.remove(index); +}} +fn overlaps(s:&State,vm:usize,start:u64,size:u64)->bool {s.regions.iter().any(|r|r.vm==vm&&r.startResult { + let mut state=vm.state.borrow_mut(); + if state.fault==Some(vm.id) {return Err(EIO)} + if overlaps(&state,vm.id,address,size) {return Err(ENOSPC)} + state.regions.push(Region {vm:vm.id,start:address,size,object}); + Ok(Mapping {vm:vm.id,address,size,object,state:vm.state.clone()}) +} +struct ObjectRef {id:u64,size:usize} +impl ObjectRef { + fn map_mode(&self,vm:&Vm,range:Range,align:u64,guard:bool,lowest:bool)->Result { + if vm.id==0 {assert!(guard,"canonical guard must be preserved");vm.state.borrow_mut().attempts+=1;} + let length=self.size as u64+if guard {16}else{0}; + let mut used:Vec<_>=vm.state.borrow().regions.iter().filter(|r|r.vm==vm.id).cloned().collect();used.sort_by_key(|r|r.start); + let mut cursor=range.start;let mut holes=Vec::new(); + for r in used { + if r.start>cursor {holes.push(cursor..r.start.min(range.end));} + cursor=cursor.max(r.start+r.size); + } + if cursorclaim(vm,address,length,self.id),None=>Err(ENOSPC)} + } + fn map_into_range(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:bool)->Result { + self.map_mode(vm,range,align,guard,false) + } +} +struct KernelObject {obj:ObjectRef,mapping:Option} +impl KernelObject { + fn backing(dev:&Device,size:usize,_cpu:CpuMap)->Result {let mut s=dev.state.borrow_mut();s.allocations+=1;Ok(Self {obj:ObjectRef {id:s.allocations,size},mapping:None})} + fn map_alias(&self,vm:&Vm,address:u64,_prot:u8)->Result {claim(vm,address,self.obj.size as u64,self.obj.id)} + fn alias_in_lowest(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:usize)->Result {self.obj.map_mode(vm,range,align,guard!=0,true)} +} +struct Allocator<'a> {dev:&'a Device,uat:&'a Uat} +impl Allocator<'_> { +'''+method+''' +} +fn case(obstacles:&[(usize,u64,u64)],fault:Option,size:usize,expected:std::result::Result,attempts:usize) { + let state=Rc::new(RefCell::new(State::default())); + for (vm,start,len) in obstacles {state.borrow_mut().regions.push(Region {vm:*vm,start:*start,size:*len,object:0});} + state.borrow_mut().fault=fault; + let dev=Device {state:state.clone()};let uat=Uat {lower:Vm {id:0,state:state.clone()}}; + let first=Vm {id:1,state:state.clone()};let second=Vm {id:2,state:state.clone()}; + let alloc=Allocator {dev:&dev,uat:&uat}; + let result=alloc.lower_with_aliases(size,16,3,CpuMap::WriteCombined,[&first,&second]); + match (expected,result) { + (Ok(address),Ok((object,aliases)))=>{ + assert_eq!(object.mapping.as_ref().unwrap().iova(),address); + assert!(aliases.iter().all(|m|m.iova()==address&&m.object==object.obj.id)); + assert_eq!(state.borrow().regions.iter().filter(|r|r.object!=0).count(),3); + drop(aliases);drop(object); + }, + (Err(expected),Err(actual))=>assert_eq!(actual,expected), + _=>panic!("unexpected common-address placement result"), + } + assert_eq!(state.borrow().regions.len(),obstacles.len(),"failed attempts leak no reservations or aliases"); + assert_eq!(state.borrow().allocations,if size==0||size%16!=0 {0}else{1},"all retries reuse one backing"); + assert_eq!(state.borrow().attempts,attempts); +} +fn exhaustive_common_holes() { + for canonical in 0u32..64 {for global in 0u32..64 {for client in 0u32..64 { + let masks=[canonical,global,client]; + let available=(0..4).any(|page|canonical&(7<{ + assert!(available,"allocator found a false common hole"); + let address=object.mapping.as_ref().unwrap().iova();assert_eq!(address%16,0);let page=address/16;assert!(page<4); + assert!(canonical&(7<{assert_eq!(error,ENOSPC);assert!(!available,"allocator skipped a common hole");}, + } + assert_eq!(state.borrow().allocations,1);assert_eq!(state.borrow().regions.len(),original); + }}} + println!("shared aliases: all262144 three-VM occupancy combinations match the independent common-hole oracle"); +} +fn main() { + case(&[],None,64,Ok(0),1); + case(&[(1,0,16)],None,64,Ok(16),2); + case(&[(2,0,16)],None,64,Ok(16),2); + case(&[(1,0,16),(2,64,16)],None,64,Ok(80),3); + case(&[(0,0,16)],None,64,Ok(16),1); + case(&[(2,0,256)],None,64,Err(ENOSPC),1); + case(&[(2,0,176)],None,64,Ok(176),2); + case(&[],Some(1),64,Err(EIO),1); + case(&[],Some(2),64,Err(EIO),1); + case(&[],Some(0),64,Err(EIO),1); + case(&[],None,256,Err(ENOSPC),1); + case(&[],None,0,Err(EINVAL),0); + case(&[],None,15,Err(EINVAL),0); + // Best-fit initially selects the small late hole, occupied in the client. + // The larger earlier canonical hole must remain eligible after that collision. + case(&[(0,128,16),(2,144,112)],None,64,Ok(0),2); + case(&[(1,0,256)],None,64,Err(ENOSPC),1); + case(&[(0,128,128),(2,0,128)],None,64,Err(ENOSPC),2); + println!("shared aliases: 16 best-fit, collision, rollback, lifetime, backing-reuse, guard and error controls pass"); + exhaustive_common_holes(); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'shared-alias-controls')],check=True) + subprocess.run([str(root/'shared-alias-controls')],check=True) + +if __name__ == '__main__': + with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: + admission_tests(Path(temporary), DRIVER) + queue_tests(Path(temporary), DRIVER) + shared_alias_tests(Path(temporary), DRIVER) From c9f72dc51116c12b3aa840185eee5a45f4b298b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 04:46:51 +1100 Subject: [PATCH 02/11] drm/asahi: carry G17 pool-release recovery and stamp-wait fixup to 7.3 Port Dj's "drm/asahi: wait for pending G17 compute pool releases" from omacom/linux-aurora#151, together with our fixup that treats a stamp-waiting released queue as admission backpressure. The first keeps observing pending pool-release consumers even when no new GPU event arrives and waits for that capacity outside device locks. The second covers the stamp-retirement interval before a released queue can become reusable. The killed/quarantined QID recyclers are already present in the 7.3 base; do not duplicate or weaken those witnesses. Preserve the normal admission, physical QID identity and pool-release contracts. Source-derived host controls include four negative controls. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 13 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 34 +++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 +- tools/asahi/g17-pool-release-tests.py | 176 ++++++++++++++++++++ 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/g17-pool-release-tests.py diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 6e69ddd56b43e..cf52e99b73033 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -626,6 +626,19 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + + /// An exited owner's finished command waits only for its completion stamp, + /// which `retirement::STAMP_FALLBACK_NS` bounds. Polling retires it without + /// a new GPU event, after which the queue can be reused. + pub(crate) fn release_settling(&self) -> bool { + self.owner.is_none() && self.released && self.retirement_polling() + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 40d2a54b59e63..4a8507b494bcf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -273,6 +273,22 @@ impl Registry { Ok(freed) } + /// A retained queue in a pool-release transaction, or one whose exited + /// owner's last command waits for its completion stamp, temporarily cannot + /// bind. Distinguish those states from permanent physical-identity + /// exhaustion. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC + && self.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_pending() || queue.release_settling()) + { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -294,7 +310,8 @@ impl Registry { let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -595,6 +612,8 @@ impl Backend { } fn build_compute(&self, required: usize) -> Result { + const POOL_RELEASE_WAIT_NS: u64 = 200_000_000; + let mut pool_release_wait = None; loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -624,6 +643,19 @@ impl Backend { Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // The event worker owns release observation and + // drops vacated aliases/backing off this mutex. + let since = *pool_release_wait.get_or_insert(now_ns()); + drop(state); + drop(lease); + self.shared.queue_events(); + if now_ns().saturating_sub(since) >= POOL_RELEASE_WAIT_NS { + return Err(EBUSY); + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; drop(state); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 8aada512d92b8..5db6e7ad0c4c0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -419,7 +419,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } diff --git a/tools/asahi/g17-pool-release-tests.py b/tools/asahi/g17-pool-release-tests.py new file mode 100644 index 0000000000000..31c1788d74190 --- /dev/null +++ b/tools/asahi/g17-pool-release-tests.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production pool-release admission and worker wake predicates. + +Firmware and allocation boundaries are host stubs. This is not GPU execution. +Both original failure modes must fail the same state-transition assertions. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + compute = (ROOT / 'queue/compute.rs').read_text() + events = (ROOT / 'runtime/events.rs').read_text() + predicate = '.any(|queue| queue.retirement_polling() || queue.pool_release_pending())' + assert events.count(predicate) == 1 + methods = '\n'.join(item(runtime, name) for name in [ + 'fn compute_admission_error(', 'pub(super) fn reserve_compute(']).replace('pub(super) ', '') + pending = item(compute, 'pub(crate) fn pool_release_pending(') + '\n' + \ + item(compute, 'pub(crate) fn release_settling(') + return r''' +#![allow(dead_code)] +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENOSPC: Error = -28; +const EAGAIN: Error = -11; +const FIRST_COMPUTE_QID: u8 = 4; +#[derive(Clone, Copy, PartialEq, Eq)] +enum DataMaster { Compute } +#[derive(Clone, Copy)] +enum PoolPhase { Populated, ReleaseSent(u32), Vacant } +mod compute { + use super::*; + pub(super) struct Queue { + pub owner: Option, + pub pool_phase: PoolPhase, + pub retire_pending: bool, + pub released: bool, + } + impl Queue { + pub(super) fn owner(&self) -> Option { self.owner } + pub(super) fn retirement_polling(&self) -> bool { self.retire_pending } +PENDING + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Id { qid: u8 } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ComputeReservation { slot: usize, id: Id, owner: u64 } +struct ComputeEntry { reservation: ComputeReservation, queue: Option> } +struct QueueIds { free: bool, fail: Option } +impl QueueIds { + fn reserve(&mut self, _: u64, _: DataMaster, preferred: Option) -> Result { + if let Some(error) = self.fail { return Err(error); } + if !self.free { return Err(ENOSPC); } + self.free = false; + Ok(Id { qid: preferred.unwrap() }) + } +} +struct Identities { next: u64 } +impl Identities { + fn reserve(&mut self, count: u64) -> Result> { + let first = self.next; self.next += count; + Ok(first..self.next) + } +} +struct Registry { + compute: Vec>, + identities: Identities, + qids: QueueIds, +} +impl Registry { +METHODS + fn poll(&self) -> bool { + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref())PREDICATE + } +} +fn state(phase: PoolPhase, free: bool) -> Registry { + let mut entries = Vec::new(); + for slot in 0..4 { + entries.push(Some(ComputeEntry { + reservation: ComputeReservation { slot, id: Id { qid: slot as u8 }, owner: slot as u64 + 1 }, + queue: Some(Box::new(compute::Queue { owner: None, pool_phase: phase, retire_pending: false, released: true })), + })); + } + entries.push(None); + Registry { compute: entries, identities: Identities { next: 100 }, qids: QueueIds { free, fail: None } } +} +fn main() { + // Captured failure: four idle released queues, full physical namespace, + // control-release cursor still awaiting observation, no retirement event. + let mut registry = state(PoolPhase::ReleaseSent(3), false); + assert!(registry.poll(), "release needs a worker wake without new GPU events"); + for _ in 0..8 { + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "pending release is temporary admission backpressure"); + assert!(registry.compute[4].is_none(), "failed admission must not install a reservation"); + assert!(!registry.qids.free); + } + // Only the firmware witness may move the retained graph to vacant. + for entry in registry.compute.iter_mut().flatten() { + entry.queue.as_mut().unwrap().pool_phase = PoolPhase::Vacant; + } + assert!(!registry.poll(), "settled idle queues must stop polling"); + assert_eq!(registry.compute_admission_error(ENOSPC), ENOSPC); + for phase in [PoolPhase::Populated, PoolPhase::Vacant] { + let mut registry = state(phase, false); + assert!(!registry.poll()); + assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "real exhaustion remains terminal"); + } + // Pending releases do not throttle spare physical capacity or hide errors. + let mut registry = state(PoolPhase::ReleaseSent(u32::MAX), true); + let reservation = registry.reserve_compute(200).unwrap(); + assert_eq!(reservation.slot, 4); + assert_eq!(reservation.id.qid, FIRST_COMPUTE_QID); + assert!(registry.compute[4].is_some()); + for error in [EINVAL, -5, -12, EAGAIN] { + let mut registry = state(PoolPhase::ReleaseSent(3), false); + registry.qids.fail = Some(error); + assert_eq!(registry.reserve_compute(200), Err(error)); + } + let mut registry = state(PoolPhase::Populated, false); + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().retire_pending = true; + assert!(registry.poll(), "existing command-retirement polling remains enabled"); + // An exited owner's command waiting for its stamp settles without a new event. + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "a settling released queue is temporary backpressure"); + // A command still owned by a live client is not a release. + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().owner = Some(1); + assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "an owned retiring queue does not hide exhaustion"); + println!("PASS captured four-queue transition, settled wake removal, stamp-wait settling, full/free identity capacity and error preservation"); +} +'''.replace('PENDING', pending).replace('METHODS', methods).replace('PREDICATE', predicate) + + +def main(): + code = fixture() + with tempfile.TemporaryDirectory(prefix='g17-pool-release-') as directory: + root = Path(directory) + for mutation in ['fixed', 'no-wake', 'terminal-admission']: + current = code + if mutation == 'no-wake': + current = current.replace('queue.retirement_polling() || queue.pool_release_pending()', 'queue.retirement_polling()') + elif mutation == 'terminal-admission': + current = current.replace('.map_err(|error| self.compute_admission_error(error))?', '?') + path = root / (mutation + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if mutation == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + expected = 'release needs a worker wake' if mutation == 'no-wake' else 'pending release is temporary' + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', mutation) + + +if __name__ == '__main__': + main() From 10e9da2303fc79dc205989075855350020bf115b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 05:45:20 +1100 Subject: [PATCH 03/11] drm/asahi: hand abandoned compute growth back to witness settlement Drop the outer preparation lease before the growth handoff, allowing recovery to close and join that epoch without waiting on a nested gate entrant. Record the stable physical QID and release phase before any following fallible step. Every growth failure/timeout/epoch exit relinquishes that builder ownership, leaving the published release and old backing for worker consumption even if the logical owner already closed. Uninstalled failed graphs are cancelled and destroyed outside the device mutex. Preserve both old and tentative backing when row/header rearm fails; move ownership only after all fallible writes. Keep consumed-release vacancy under collector pressure rather than dropping mappings under the device mutex. Add native preparation-join KUnit, 128 concurrent production gate handoffs, production growth abort/commit/close/rearm/witness/collector controls and four negative mutations. No asynchronous wake or performance series is imported. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/freelist.rs | 16 +- drivers/gpu/drm/asahi/g17/preparation.rs | 19 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 44 ++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 45 +++++- tools/asahi/g17-growth-gate-tests.py | 83 ++++++++++ tools/asahi/g17-growth-tests.py | 171 +++++++++++++++++++++ 6 files changed, 360 insertions(+), 18 deletions(-) create mode 100644 tools/asahi/g17-growth-gate-tests.py create mode 100644 tools/asahi/g17-growth-tests.py diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 6e677e6f761b6..ca191ba5bd35d 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -103,7 +103,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - fn page_list_va(&self) -> u64 { + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -525,6 +525,18 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } + self.write_pages(&pages)?; + self.pages = Some(pages); + Ok(()) + } + /// Preserve both sets of pages on failure. The caller owns the consumed + /// release and returns old pages for destruction outside the device mutex. + pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { + self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; + core::mem::swap(&mut self.pages, pages); + Ok(()) + } + fn write_pages(&self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -543,9 +555,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); - self.pages = Some(pages); Ok(()) } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 8563beda979e6..b10b81c2d8e14 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -184,3 +184,22 @@ impl Drop for Lease { } } } + +#[kunit_tests(asahi_g17_growth_gate)] +mod growth_gate_tests { + use super::*; + #[test] + fn recovery_can_join_between_growth_admissions() { + let gate = Gate::new().unwrap(); + let outer = gate.try_enter().unwrap(); + // The growth handoff must drop this lease before enter can wait. + drop(outer); + gate.close(); + assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); + gate.wait_drained(); + assert!(gate.reopen()); + let growth = gate.enter(false).unwrap(); + assert!(growth.is_current()); + drop(growth); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cf52e99b73033..54ed90c59392f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -453,6 +453,14 @@ pub(crate) enum PoolPhase { Vacant, } +/// Builder ownership of one physical release transaction. The preparation +/// lease pins this registry until abort/commit; owner close may remove lookups. +#[derive(Clone, Copy)] +pub(crate) struct Growth { + qid: u8, + phase: PoolPhase, +} + pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; @@ -1325,11 +1333,7 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { - if blocks < self.growth_blocks { return Err(EAGAIN); } - self.growth_blocks = blocks; - return Ok(false); - } + if self.growing { return Err(EAGAIN); } if self.quarantined || self.released || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) @@ -1355,16 +1359,34 @@ impl Queue { return Err(EAGAIN); } } + if alias.is_none() { return Err(EINVAL); } + let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - let old_alias = binding.pool_alias.take(); - let old_pages = self.pool.take_pages(); - self.pool.rearm(pages.take().ok_or(EINVAL)?)?; - init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; - binding.pool_alias = alias.take(); + // No ownership moves before every fallible descriptor/header operation. + // The old release is consumed; no queue can use this row until commit. + init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + self.pool.replace_pages(pages)?; + let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + } + + pub(crate) fn growth_token(&self) -> Option { + self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + } + + /// Transfer an abandoned builder's release back to the event worker. Do not + /// revoke the published release or free backing here: vacancy still requires + /// its exact control-consumer witness. A different transaction is untouched. + pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { + if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { + return Ok(None); + } + self.growing = false; + self.growth_blocks = 0; + self.release_owner().map(Some) } /// Exact host identity remains valid even when several commands share an ioctl ID. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4a8507b494bcf..720d00066550e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -265,6 +265,7 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -583,7 +584,8 @@ impl Backend { let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; let started = now_ns(); - loop { + let mut growth = None; + let result = (|| loop { let result = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -592,9 +594,15 @@ impl Backend { firmware.scan_compute(None, &mut deferred)?; let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; if queue.pool_fits(required) { return Ok(None); } - if queue.begin_growth(&firmware.init, required)? { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + if growth.is_none() { + let notify = queue.begin_growth(&firmware.init, required)?; + // Capture physical identity before any following fallible + // operation. Close can remove the owner-keyed lookup. + growth = queue.growth_token(); + if notify { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); @@ -608,7 +616,31 @@ impl Backend { } Err(error) => return Err(error), } + })(); + if result.is_err() { + if let Some(token) = growth { + let mut retired = None; + let mut state = self.shared.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + for index in 0..firmware.queues.compute.len() { + let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; + let Some(queue) = entry.queue.as_deref_mut() else { continue; }; + if queue.abort_growth(token)? == Some(true) { + // Only an uninstalled graph can return true. Cancel + // its unpublished QID, then destroy it off-lock. + firmware.queues.qids.cancel_unpublished(entry.reservation.id)?; + retired = entry.queue.take(); + firmware.queues.compute[index] = None; + break; + } + } + } + drop(state); + drop(retired); + self.shared.queue_events(); + } } + result } fn build_compute(&self, required: usize) -> Result { @@ -638,7 +670,7 @@ impl Backend { } match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { @@ -822,6 +854,9 @@ impl DeferredBatch { drop(release); } } + pub(super) fn vacated_room(&self) -> bool { + self.vacated.len() < self.vacated.capacity() + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); diff --git a/tools/asahi/g17-growth-gate-tests.py b/tools/asahi/g17-growth-gate-tests.py new file mode 100644 index 0000000000000..ad72c0b1c2bc6 --- /dev/null +++ b/tools/asahi/g17-growth-gate-tests.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Close recovery exactly between the production growth handoff and re-entry.""" +from pathlib import Path +import os +import re +import subprocess +import tempfile +ROOT=Path(__file__).resolve().parents[2]/'drivers/gpu/drm/asahi/g17' + + +def item(text,sig): + start=text.index(sig);begin=text.index('{',start);depth=1;end=begin+1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + + +def fixture(): + gate=(ROOT/'preparation.rs').read_text() + methods='\n'.join(item(gate,sig) for sig in ['pub(super) fn try_enter(', 'pub(super) fn enter(', 'fn invalidate(', 'pub(super) fn close(', 'pub(super) fn wait_drained(', 'pub(super) fn reopen(']).replace('pub(super) ','') + handoff=re.search(r'Rebind::Grow => \{([^}]+)\}',(ROOT/'runtime.rs').read_text()).group(1) + return r''' +use std::sync::{Arc,Mutex as StdMutex,Condvar as StdCondvar,atomic::{AtomicU64,Ordering},mpsc}; +type Result=std::result::Result; +const ENODEV:i32=-19;const EAGAIN:i32=-11;const ENOSPC:i32=-28;const ERESTARTSYS:i32=-512; +const COUNT_BITS:u32=20;const COUNT_MASK:u64=(1<(Option>); +struct Mutex(StdMutex<()>); +impl Mutex{fn lock(&self)->Guard<'_>{Guard(Some(self.0.lock().unwrap()))}} +struct CondVar(StdCondvar); +impl CondVar { + fn wait(&self,g:&mut Guard<'_>){g.0=Some(self.0.wait(g.0.take().unwrap()).unwrap());} + fn wait_interruptible(&self,g:&mut Guard<'_>)->bool{self.wait(g);false} + fn notify_all(&self){self.0.notify_all();} +} +struct Gate{state:AtomicU64,wait:Mutex,changed:CondVar} +struct Lease{gate:Arc,epoch:u64} +impl Gate { +''' + methods + '\n}\n' + item(gate,'impl Drop for Lease') + r''' +struct Backend {gate:Arc,ready:mpsc::Sender<()>,proceed:StdMutex>} +impl Backend { + fn handoff(&self,required:usize)->Result{ + let lease=self.gate.try_enter()?;let state=(); +''' + handoff + r''' + } + fn grow_compute(&self,_:usize)->Result { + self.ready.send(()).unwrap();self.proceed.lock().unwrap().recv().unwrap(); + let _growth=self.gate.enter(false)?; + Ok(()) + } +} +fn main(){for _ in 0..128 { + let gate=Arc::new(Gate{state:AtomicU64::new(EPOCH),wait:Mutex(StdMutex::new(())),changed:CondVar(StdCondvar::new())}); + let (ready_tx,ready_rx)=mpsc::channel();let (go_tx,go_rx)=mpsc::channel(); + let b=Backend{gate:gate.clone(),ready:ready_tx,proceed:StdMutex::new(go_rx)}; + let builder=std::thread::spawn(move||b.handoff(40)); + ready_rx.recv().unwrap();gate.close(); + assert_eq!(gate.state.load(Ordering::Acquire)&COUNT_MASK,0,"outer growth lease blocks recovery join"); + assert!(matches!(gate.try_enter(),Err(EAGAIN))); + go_tx.send(()).unwrap(); + let recovery=std::thread::spawn(move||{gate.wait_drained();assert!(gate.reopen());}); + recovery.join().unwrap();builder.join().unwrap().unwrap(); +} + println!("PASS 128 concurrent production growth handoffs: recovery closes between entries, joins builder, reopens and growth completes");} +''' + + +def main(): + source=fixture();assert 'drop(lease);' in source + with tempfile.TemporaryDirectory(prefix='g17-growth-gate-') as tmp: + for name,code in [('fixed',source),('retain-outer-lease',source.replace('drop(lease);',''))]: + path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021','-Awarnings',str(path),'-o',str(binary)],check=True) + run=subprocess.run([str(binary)],capture_output=True,text=True,timeout=15) + if name=='fixed': + assert run.returncode==0,run.stderr + print(run.stdout.strip()) + else: + assert run.returncode!=0 and 'outer growth lease' in run.stderr,run.stderr + print('PASS negative control:',name) +if __name__=='__main__':main() diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py new file mode 100644 index 0000000000000..5ee41ce998e62 --- /dev/null +++ b/tools/asahi/g17-growth-tests.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Production growth abort/commit/vacancy at controlled allocation/MMIO boundaries.""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' +def item(source, signature): + start=source.index(signature);begin=source.index('{',start);depth=1;end=begin+1 + while depth: + depth+=(source[end]=='{')-(source[end]=='}');end+=1 + return source[start:end] + + +def fixture(): + source = (ROOT / 'queue/compute.rs').read_text() + text = r''' +#![allow(dead_code)] +use std::sync::{Arc,atomic::{AtomicU32,AtomicUsize,Ordering}}; +type Result = std::result::Result; +const EIO:i32=-5;const EBUSY:i32=-16;const EINVAL:i32=-22; +fn now_ns()->u64{123} +struct Context {live:Arc} +impl Context {fn new(live:&Arc)->Arc{live.fetch_add(1,Ordering::SeqCst);Arc::new(Self{live:live.clone()})}} +impl Drop for Context {fn drop(&mut self){self.live.fetch_sub(1,Ordering::SeqCst);}} +struct Binding {context:Arc,pool_alias:Option,descriptors:Option} +struct Previous {context:Arc,binding:Option,publication:Option<(u64,u64)>} +#[derive(Clone,Copy,PartialEq)] enum PoolPhase{Populated,ReleaseSent(u32),Vacant} +struct Fifo {len:usize} +struct Graph; +impl Graph {fn set_owner(&mut self,_:&Context)->Result{Ok(())}} +struct Queue { + owner:Option,installed:bool,growing:bool,growth_blocks:usize,free_list_generation:u64, + pool_phase:PoolPhase,pool:Pool,binding:Option,previous:Option,graph:Graph, + active:Fifo,replays:Fifo,quarantined:bool,released:bool,retirement_ready:bool, + retired_by_teardown:bool,released_at:u64,exit:Option<()>,needs_fresh_backing:bool, +} +fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth_blocks:0, + free_list_generation:0,pool_phase:PoolPhase::Populated,pool:Pool{pages:Some(Pages(10)),fail:false}, + binding:Some(Binding{context:Context::new(live),pool_alias:Some(1),descriptors:Some(2)}),previous:None,graph:Graph, + active:Fifo{len:0},replays:Fifo{len:0},quarantined:false,released:false,retirement_ready:true, + retired_by_teardown:false,released_at:0,exit:None,needs_fresh_backing:false}} +impl Queue { +''' + item(source,'pub(crate) fn release_owner(') + item(source,'pub(crate) fn reusable(') + '\n}\n' + methods = '\n'.join(item(source, sig) for sig in ['pub(crate) fn begin_growth(', 'pub(crate) fn finish_growth(', 'pub(crate) fn growth_token(', 'pub(crate) fn abort_growth(', 'pub(crate) fn observe_vacancy(']) + replace = item((ROOT / 'freelist.rs').read_text(), 'pub(crate) fn replace_pages(') + observe = item((ROOT / 'runtime.rs').read_text(), 'pub(super) fn observe_pool_releases(').replace('pub(super) ', '').replace('super::initdata::', 'initdata::') + return text + r''' +const EAGAIN:i32=-11;const EFAULT:i32=-14; +static CONSUMER:AtomicU32=AtomicU32::new(0); +static PRODUCER:AtomicU32=AtomicU32::new(3); +struct Pages(u64); +impl Pages {fn page_list_va(&self)->u64{self.0}} +mod mmu {pub type KernelMapping=u64;} +mod g17 { + pub mod freelist {pub(crate) use crate::Pages; pub fn control_consumed(c:u32,_:u32,t:u32)->crate::Result{Ok(c==t)}} + pub mod initdata { + pub struct InitData {pub fail:bool} + impl InitData { + pub fn control_consumer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::CONSUMER)} + pub fn control_producer(&self)->crate::Result<&crate::AtomicU32>{Ok(&crate::PRODUCER)} + pub fn initialize_compute_pool_blocks(&self,_:u16,_:u64,_:usize)->crate::Result{if self.fail{Err(crate::EIO)}else{Ok(())}} + } + } +} +use std::sync::atomic::fence; +use g17::initdata; +struct Pool {pages:Option,fail:bool} +impl Pool { + fn id(&self)->u16{1} + fn release_record(&self,_:u64)->u64{0} + fn take_pages(&mut self)->Option{self.pages.take()} + fn write_pages(&self,_:&Pages)->Result{if self.fail{Err(EIO)}else{Ok(())}} +''' + replace + r''' +} +enum ControlRecord {FreeListRelease(u64)} +struct Rings; +impl Rings {fn publish_control(_:&initdata::InitData,_:&ControlRecord)->Result{Ok(3)}} +struct Vacated {_alias:Option,_pages:Option,_descriptors:Option,_ring:Option} +impl Graph {fn vacate_ring(&mut self)->Option{Some(1)}} +''' + '#[derive(Clone, Copy)]\n' + item(source, 'pub(crate) struct Growth') + r''' +impl Queue { + fn qid(&self)->u8{7} +''' + methods + r''' +} +struct Entry {queue:Option>} +struct Registry {compute:Vec>} +struct DeferredBatch {vacated:Vec,capacity:usize} +impl DeferredBatch { + fn vacated_room(&self)->bool{self.vacated.len() Date: Wed, 21 Oct 2026 05:49:59 +1100 Subject: [PATCH 04/11] drm/asahi: borrow the compute pool mutably for header replacement KernelObject::write requires mutable ownership. Preserve that requirement in the extracted helper and its host allocation boundary. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/freelist.rs | 2 +- tools/asahi/g17-growth-tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index ca191ba5bd35d..6b4522cea9a15 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -536,7 +536,7 @@ impl ComputePool { core::mem::swap(&mut self.pages, pages); Ok(()) } - fn write_pages(&self, pages: &Pages) -> Result { + fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index 5ee41ce998e62..e97e29d248e2c 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -72,7 +72,7 @@ def fixture(): fn id(&self)->u16{1} fn release_record(&self,_:u64)->u64{0} fn take_pages(&mut self)->Option{self.pages.take()} - fn write_pages(&self,_:&Pages)->Result{if self.fail{Err(EIO)}else{Ok(())}} + fn write_pages(&mut self,_:&Pages)->Result{if self.fail{Err(EIO)}else{Ok(())}} ''' + replace + r''' } enum ControlRecord {FreeListRelease(u64)} From 04358c1ebe86cfbfdc926cfb41ff3e3549248279 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 06:24:48 +1100 Subject: [PATCH 05/11] drm/asahi: hold failed growth reservations until aliases have gone Leave the uninstalled queue's empty registry entry and unpublished QID reserved while dropping its graph off the device mutex. Only cancel that exact constructor reservation after all aliases are gone, while the preparation lease still pins the session. This retains the existing drop-before-reuse construction obligation on the new growth-abort path. Add an actual registry KUnit test for blocked reuse and stale reservation identity, and 512 production cleanup controls with mutations for early QID return and mapping destruction under the device mutex. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/runtime.rs | 44 ++++++++-- tools/asahi/g17-uninstalled-cleanup-tests.py | 90 ++++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 tools/asahi/g17-uninstalled-cleanup-tests.py diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 720d00066550e..5a2ffc63e4023 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -389,6 +389,26 @@ pub(super) enum Rebind { Fresh, } + +#[kunit_tests(asahi_g17_uninstalled_cleanup)] +mod uninstalled_cleanup_tests { + use super::*; + #[test] + fn empty_reservation_prevents_reuse_until_explicit_cancellation() { + let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let first = registry.reserve_compute(1).unwrap(); + assert!(registry.reserved(first).is_ok()); + assert!(registry.reserve_compute(1).is_err()); + let second = registry.reserve_compute(2).unwrap(); + assert_ne!(first.id.qid(), second.id.qid()); + registry.cancel_compute(first).unwrap(); + assert!(registry.reserved(first).is_err()); + let next = registry.reserve_compute(3).unwrap(); + assert_eq!(first.id.qid(), next.id.qid()); + assert!(first.id != next.id); + } +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -567,6 +587,20 @@ impl Backend { result } + /// The caller keeps its preparation lease through this two-phase rollback. + /// An empty registry entry prevents reusing the QID/free-list identity while + /// its old aliases still occupy canonical or client addresses. Recovery joins + /// this lease before removing the session, even if its epoch was invalidated. + fn finish_uninstalled_compute( + &self, retired: Option<(ComputeReservation, KBox)>, + ) -> Result { + let Some((reservation, queue)) = retired else { return Ok(()); }; + drop(queue); + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.queues.cancel_compute(reservation) + } + fn grow_compute(&self, required: usize) -> Result { use kernel::time::{delay::fsleep, Delta}; { @@ -626,17 +660,15 @@ impl Backend { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; let Some(queue) = entry.queue.as_deref_mut() else { continue; }; if queue.abort_growth(token)? == Some(true) { - // Only an uninstalled graph can return true. Cancel - // its unpublished QID, then destroy it off-lock. - firmware.queues.qids.cancel_unpublished(entry.reservation.id)?; - retired = entry.queue.take(); - firmware.queues.compute[index] = None; + // Keep the empty entry and QID reserved until all + // graph aliases have been destroyed outside the lock. + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); break; } } } drop(state); - drop(retired); + self.finish_uninstalled_compute(retired)?; self.shared.queue_events(); } } diff --git a/tools/asahi/g17-uninstalled-cleanup-tests.py b/tools/asahi/g17-uninstalled-cleanup-tests.py new file mode 100644 index 0000000000000..d2377626a4e37 --- /dev/null +++ b/tools/asahi/g17-uninstalled-cleanup-tests.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""The production cleanup boundary must unmap before returning an identity.""" +from pathlib import Path +import os +import subprocess +import tempfile +ROOT=Path(__file__).resolve().parents[2]/'drivers/gpu/drm/asahi/g17' + + +def item(source,sig): + start=source.index(sig);begin=source.index('{',start);depth=1;end=begin+1 + while depth: + depth+=(source[end]=='{')-(source[end]=='}');end+=1 + return source[start:end] + + +def fixture(): + runtime=(ROOT/'runtime.rs').read_text() + helper=item(runtime,'fn finish_uninstalled_compute(') + # Growth must retain the empty constructor entry until the helper consumes it. + abort=item(runtime,'fn grow_compute(') + assert 'retired = entry.queue.take().map(|queue| (entry.reservation, queue));' in abort + assert 'self.finish_uninstalled_compute(retired)?;' in abort + assert 'cancel_unpublished' not in abort + return r''' +use std::sync::{Arc,Mutex,atomic::{AtomicBool,Ordering}}; +use std::ops::{Deref,DerefMut}; +type Result = std::result::Result; +type KBox = Box; +const ENODEV:i32=-19; +static MAPPED:AtomicBool=AtomicBool::new(false); +static RESERVED:AtomicBool=AtomicBool::new(false); +static LOCKED:AtomicBool=AtomicBool::new(false); +#[derive(Clone,Copy)] struct ComputeReservation(u64); +struct Queue; +impl Drop for Queue {fn drop(&mut self){ + assert!(!LOCKED.load(Ordering::SeqCst),"mapping destruction under device mutex"); + assert!(RESERVED.load(Ordering::SeqCst),"identity returned before aliases disappeared"); + MAPPED.store(false,Ordering::SeqCst); +}} +mod compute {pub(crate) use crate::Queue;} +struct Registry; +impl Registry {fn cancel_compute(&mut self,res:ComputeReservation)->Result{ + assert_eq!(res.0,7); + assert!(!MAPPED.load(Ordering::SeqCst),"identity returned before aliases disappeared"); + assert!(RESERVED.swap(false,Ordering::SeqCst));Ok(()) +}} +struct Firmware{queues:Registry} +struct Guard<'a>(std::sync::MutexGuard<'a,Option>>); +impl Deref for Guard<'_>{type Target=Option>;fn deref(&self)->&Self::Target{&self.0}} +impl DerefMut for Guard<'_>{fn deref_mut(&mut self)->&mut Self::Target{&mut self.0}} +impl Drop for Guard<'_>{fn drop(&mut self){LOCKED.store(false,Ordering::SeqCst);}} +struct State(Mutex>>); +impl State{fn lock(&self)->Guard<'_>{let g=self.0.lock().unwrap();assert!(!LOCKED.swap(true,Ordering::SeqCst));Guard(g)}} +struct Shared{state:State} +struct Backend{shared:Arc} +impl Backend { +''' +helper+r''' +} +fn main(){ + let b=Backend{shared:Arc::new(Shared{state:State(Mutex::new(Some(Box::new(Firmware{queues:Registry}))))})}; + b.finish_uninstalled_compute(None).unwrap(); + for _ in 0..512 { + MAPPED.store(true,Ordering::SeqCst);RESERVED.store(true,Ordering::SeqCst); + b.finish_uninstalled_compute(Some((ComputeReservation(7),Box::new(Queue)))).unwrap(); + assert!(!MAPPED.load(Ordering::SeqCst));assert!(!RESERVED.load(Ordering::SeqCst)); + } + println!("PASS 512 production uninstalled cleanups: graph aliases drop off-lock while the identity remains reserved; only then can QID cancellation make it reusable"); +} +''' + + +def main(): + source=fixture() + cases=[('fixed',source),('cancel-before-alias-drop',source.replace('drop(queue);','core::mem::forget(queue);')), + ('drop-under-device-mutex',source.replace(' drop(queue);\n','').replace(' let mut state = self.shared.state.lock();',' let mut state = self.shared.state.lock();\n drop(queue);'))] + with tempfile.TemporaryDirectory(prefix='g17-uninstalled-') as tmp: + for name,code in cases: + path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021','-Awarnings',str(path),'-o',str(binary)],check=True) + run=subprocess.run([str(binary)],capture_output=True,text=True,timeout=15) + if name=='fixed': + assert run.returncode==0,run.stderr + print(run.stdout.strip()) + else: + assert run.returncode!=0,name + assert ('identity returned before aliases disappeared' if name=='cancel-before-alias-drop' else 'mapping destruction under device mutex') in run.stderr,run.stderr + print('PASS negative control:',name) +if __name__=='__main__':main() From d765e50ada90e7f00a4ad608c9ff027d0a07a82a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 06:29:19 +1100 Subject: [PATCH 06/11] drm/asahi: drop tentative growth aliases before returning a fresh QID The failed growth frame can own replacement pages and client aliases outside the uninstalled graph. Drop those mappings off-lock before the common cleanup can cancel the graph's reservation. Retain the preparation lease across both phases. Exercise the exact production growth cleanup tail as well as its common helper; 512 cycles per boundary reject live tentative aliases, premature QID reuse and mapping destruction under the device mutex. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +++ tools/asahi/g17-uninstalled-cleanup-tests.py | 26 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 5a2ffc63e4023..b080cbe604055 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -668,6 +668,10 @@ impl Backend { } } drop(state); + // Tentative replacement mappings also belong to this builder. + // Discard them before a fresh QID can acquire a successor. + drop(alias.take()); + drop(pages.take()); self.finish_uninstalled_compute(retired)?; self.shared.queue_events(); } diff --git a/tools/asahi/g17-uninstalled-cleanup-tests.py b/tools/asahi/g17-uninstalled-cleanup-tests.py index d2377626a4e37..33d2f252eb81c 100644 --- a/tools/asahi/g17-uninstalled-cleanup-tests.py +++ b/tools/asahi/g17-uninstalled-cleanup-tests.py @@ -23,6 +23,9 @@ def fixture(): assert 'retired = entry.queue.take().map(|queue| (entry.reservation, queue));' in abort assert 'self.finish_uninstalled_compute(retired)?;' in abort assert 'cancel_unpublished' not in abort + start=abort.index(' drop(state);\n // Tentative replacement mappings') + end=abort.index('\n }',start) + cleanup=abort[start:end] return r''' use std::sync::{Arc,Mutex,atomic::{AtomicBool,Ordering}}; use std::ops::{Deref,DerefMut}; @@ -30,6 +33,8 @@ def fixture(): type KBox = Box; const ENODEV:i32=-19; static MAPPED:AtomicBool=AtomicBool::new(false); +static ALIAS:AtomicBool=AtomicBool::new(false); +static PAGES:AtomicBool=AtomicBool::new(false); static RESERVED:AtomicBool=AtomicBool::new(false); static LOCKED:AtomicBool=AtomicBool::new(false); #[derive(Clone,Copy)] struct ComputeReservation(u64); @@ -37,8 +42,11 @@ def fixture(): impl Drop for Queue {fn drop(&mut self){ assert!(!LOCKED.load(Ordering::SeqCst),"mapping destruction under device mutex"); assert!(RESERVED.load(Ordering::SeqCst),"identity returned before aliases disappeared"); + assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst),"tentative aliases still live at identity return"); MAPPED.store(false,Ordering::SeqCst); }} +struct Tentative(&'static AtomicBool); +impl Drop for Tentative{fn drop(&mut self){assert!(!LOCKED.load(Ordering::SeqCst));self.0.store(false,Ordering::SeqCst);}} mod compute {pub(crate) use crate::Queue;} struct Registry; impl Registry {fn cancel_compute(&mut self,res:ComputeReservation)->Result{ @@ -54,9 +62,18 @@ def fixture(): struct State(Mutex>>); impl State{fn lock(&self)->Guard<'_>{let g=self.0.lock().unwrap();assert!(!LOCKED.swap(true,Ordering::SeqCst));Guard(g)}} struct Shared{state:State} +impl Shared{fn queue_events(&self){}} struct Backend{shared:Arc} impl Backend { ''' +helper+r''' + fn growth_cleanup(&self)->Result{ + let mut alias=Some(Tentative(&ALIAS));let mut pages=Some(Tentative(&PAGES)); + ALIAS.store(true,Ordering::SeqCst);PAGES.store(true,Ordering::SeqCst); + let retired=Some((ComputeReservation(7),Box::new(Queue))); + let state=self.shared.state.lock(); +''' +cleanup+r''' + Ok(()) + } } fn main(){ let b=Backend{shared:Arc::new(Shared{state:State(Mutex::new(Some(Box::new(Firmware{queues:Registry}))))})}; @@ -65,8 +82,11 @@ def fixture(): MAPPED.store(true,Ordering::SeqCst);RESERVED.store(true,Ordering::SeqCst); b.finish_uninstalled_compute(Some((ComputeReservation(7),Box::new(Queue)))).unwrap(); assert!(!MAPPED.load(Ordering::SeqCst));assert!(!RESERVED.load(Ordering::SeqCst)); + MAPPED.store(true,Ordering::SeqCst);RESERVED.store(true,Ordering::SeqCst); + b.growth_cleanup().unwrap(); + assert!(!MAPPED.load(Ordering::SeqCst));assert!(!RESERVED.load(Ordering::SeqCst)); } - println!("PASS 512 production uninstalled cleanups: graph aliases drop off-lock while the identity remains reserved; only then can QID cancellation make it reusable"); + println!("PASS 512 growth-frame and 512 production uninstalled cleanups: graph aliases drop off-lock while the identity remains reserved; only then can QID cancellation make it reusable"); } ''' @@ -74,6 +94,7 @@ def fixture(): def main(): source=fixture() cases=[('fixed',source),('cancel-before-alias-drop',source.replace('drop(queue);','core::mem::forget(queue);')), + ('keep-tentative-growth-aliases',source.replace('drop(alias.take());','').replace('drop(pages.take());','')), ('drop-under-device-mutex',source.replace(' drop(queue);\n','').replace(' let mut state = self.shared.state.lock();',' let mut state = self.shared.state.lock();\n drop(queue);'))] with tempfile.TemporaryDirectory(prefix='g17-uninstalled-') as tmp: for name,code in cases: @@ -85,6 +106,7 @@ def main(): print(run.stdout.strip()) else: assert run.returncode!=0,name - assert ('identity returned before aliases disappeared' if name=='cancel-before-alias-drop' else 'mapping destruction under device mutex') in run.stderr,run.stderr + expected={'cancel-before-alias-drop':'identity returned before aliases disappeared','keep-tentative-growth-aliases':'tentative aliases still live at identity return','drop-under-device-mutex':'mapping destruction under device mutex'} + assert expected[name] in run.stderr,run.stderr print('PASS negative control:',name) if __name__=='__main__':main() From 780956abdb5fe59c76ca7343bc7f19770df658dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 10:38:30 +1100 Subject: [PATCH 07/11] drm/asahi: compose shared capacity admission with accepted recovery port Apply the shared-capacity result from fu-g17-capacity-leak-73-v2 onto 883bd87115de82816a0a6f38f66ecbb784002459. Retain the accepted recovery history and its pool-release polling, exact stamp/drain lifetime witnesses, growth lease handoff and tentative-alias-before-QID rollback order. Resolve the overlapping compute/runtime/test policy with shared capacity: closed owners and constructors remain temporary capacity pressure; all healthy shared-channel exhaustion is retryable, and item-ring refusal precedes mutation. Retain the later unpublished-binding rollback and predecessor retirement fixes. Keep the K1 absolute packet deadline/cancellation propagation and K2's reachable pool-release mutations, including the updated preparation/growth test adapter. The resulting source tree is identical to the v2 shared candidate. This linear composition starts at the accepted recovery head; it does not merge histories or duplicate recovery fixes whose equivalent shared commits already exist. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/asahi-g17-capacity.rst | 157 +++++++++ drivers/gpu/drm/asahi/g17/buffer.rs | 52 ++- drivers/gpu/drm/asahi/g17/capacity.rs | 87 +++++ drivers/gpu/drm/asahi/g17/context.rs | 39 ++- drivers/gpu/drm/asahi/g17/context/identity.rs | 9 + drivers/gpu/drm/asahi/g17/freelist.rs | 29 +- drivers/gpu/drm/asahi/g17/job.rs | 3 +- drivers/gpu/drm/asahi/g17/kick.rs | 107 +++++- drivers/gpu/drm/asahi/g17/mod.rs | 26 ++ drivers/gpu/drm/asahi/g17/object.rs | 45 ++- drivers/gpu/drm/asahi/g17/preparation.rs | 123 +++++++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 277 ++++++++++++++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 276 ++++++++++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 3 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/runtime.rs | 308 ++++++++++++++---- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 44 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 55 +++- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 8 + .../drm/asahi/g17/runtime/render/prepare.rs | 2 +- .../drm/asahi/g17/runtime/render/submit.rs | 36 +- drivers/gpu/drm/asahi/hw/t8140.rs | 4 +- drivers/gpu/drm/asahi/mmu.rs | 27 ++ tools/asahi/g17-admission-deadline-tests.py | 131 ++++++++ tools/asahi/g17-admission-order-tests.py | 89 +++++ tools/asahi/g17-capacity-fault-tests.py | 287 ++++++++++++++++ tools/asahi/g17-capacity-watch.sh | 14 + tools/asahi/g17-compute-constructor-tests.py | 118 +++++++ tools/asahi/g17-compute-deadline-tests.py | 164 ++++++++++ .../g17-compute-growth-deadline-tests.py | 129 ++++++++ tools/asahi/g17-growth-gate-tests.py | 4 +- tools/asahi/g17-pool-release-tests.py | 37 ++- tools/asahi/g17-resource-tests.py | 30 +- tools/asahi/g17-shared-pool-tests.py | 139 ++++++++ tools/asahi/g17-unpublished-tests.py | 110 +++++++ 35 files changed, 2827 insertions(+), 145 deletions(-) create mode 100644 Documentation/gpu/asahi-g17-capacity.rst create mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs create mode 100644 tools/asahi/g17-admission-deadline-tests.py create mode 100644 tools/asahi/g17-admission-order-tests.py create mode 100644 tools/asahi/g17-capacity-fault-tests.py create mode 100644 tools/asahi/g17-capacity-watch.sh create mode 100644 tools/asahi/g17-compute-constructor-tests.py create mode 100644 tools/asahi/g17-compute-deadline-tests.py create mode 100644 tools/asahi/g17-compute-growth-deadline-tests.py create mode 100644 tools/asahi/g17-shared-pool-tests.py create mode 100644 tools/asahi/g17-unpublished-tests.py diff --git a/Documentation/gpu/asahi-g17-capacity.rst b/Documentation/gpu/asahi-g17-capacity.rst new file mode 100644 index 0000000000000..68d261682f842 --- /dev/null +++ b/Documentation/gpu/asahi-g17-capacity.rst @@ -0,0 +1,157 @@ +G17 capacity diagnostics +======================= + +The read-only ``/sys/kernel/debug/asahi-g17/capacity`` file samples the live +firmware session. It does not wake firmware or reclaim resources. An absent +session reports ``online=0``. The format is diagnostic, not a stable ABI. +Counts exclude reserved identities. Sub-pool locks and atomic snapshots mean +that independent rows need not describe exactly the same instant. + +``used`` is the number of allocated entries unless stated below: + +* Physical QIDs are device-lifetime reservations, including installed queues. + An idle installed QID is reused by its existing engine, never cancelled or + retyped. Published QIDs need not return to zero after the last client exits. +* Render slot ``used`` excludes released, inactive slots that can be reassigned. + ``retained`` includes those slots; ``active``, ``quarantined`` and ``kill_held`` + explain why a released owner can still occupy capacity. A published context + kill retains its render holds until processor stop under the current policy. +* Compute slots include retained graphs and off-lock construction reservations. + ``reusable`` counts graphs immediately available for another owner. A vacant + graph still needs fresh pool backing; a populated recycled graph cannot rebind + until its pool release was consumed and it was rearmed. Per-queue rows show + these states and the command and item-ring budgets. +* QoS, execution-context, render-pool, parameter-buffer and metrics rows sample + their actual allocators. A parameter buffer can retain either a graph owner + or command references. Neither an error fence nor logical close returns a + firmware-owned reference. +* Context rows show the 63 work-state nodes and two stable render ordinals. + A context can occur in several graph rows; do not sum duplicate contexts. +* Cached-object reservations include active, returned and poisoned mappings. + Returned mappings can be reused; poisoned mappings wait for processor stop. + Reaching this cache's admission cap falls back to uncached allocation. +* Dynamic VA rows count allocator reservations including unmapped guards. The + largest hole helps distinguish fragmentation from aggregate exhaustion; + alignment and alias-VM collisions can still prevent a particular allocation. + Removal can progress during this diagnostic walk. Values are observations, + not promises that a subsequent placement will succeed. +* Refusal counters name allocation attempts, including attempts subsequently + retried successfully. They are module-wide cumulative counters; pool gauges + refer to this device. Compare counter deltas with failed submits and fences. + +Shared admission budget +----------------------- + +The 128 physical identities cover one tiling/fragment pair for each of the +58 logical execution contexts, two bootstrap reservations, and ten shared +compute channels: ``2 * 58 + 2 + 10 = 128``. Logical render owners no longer +reserve a speculative second pair. Its 16 in-flight commands share the same +physical pair; admission waits, interruptibly and with the existing two-second +job budget, when that pair is full. The DRM queued-job credit budget is retained. + +Compute sizing is validated at ioctl admission, but a physical channel is not +reserved until DRM dependencies permit the job to run. This matters for reverse +timeline chains: the host signal can be submitted only after their earlier +ioctls return. An idle compute channel can yield to another logical queue only +after exact retirement, with no pending kill, growth, quarantine, replay, +predecessor or pool release. Its original QID and ring remain installed. + +Capacity pressure waits outside device locks and preparation leases. A short +reclaim transaction takes the usual session/epoch lease, polls consumed pool +releases and may publish idle compute-pool releases. Render backing is reclaimed +only if never registered or after its free-list release was consumed, all +associated graphs are idle, +and no admission borrower or off-lock builder can use it. Backing destruction +remains deferred until after the device mutex is released; the next owner +rearms the same pool using the existing epoch/descriptor protocol. A deadline +never frees an unwitnessed resource. Terminal errors are preserved. + +Logical close is recorded before scanning installed graphs. Late compute +constructors and rearms recheck that flag, so an off-lock reservation cannot +resurrect a frontend that already closed. + +The VA diagnostic walk retains UAT ownership but drops the device mutex before +acquiring VM/TTBAT locks, so mapping destruction and its firmware flushes cannot +invert the diagnostic lock order. + +Compute close and failure ownership +----------------------------------- + +A never-installed graph can cancel its unpublished QID. An installed graph +stays allocated after close. If a detached graph was rebound and closed before +its first new publication, an empty, non-quarantined command FIFO retains its +original idle witness: no future completion exists to rehabilitate that owner. + +A superseded compute binding remains owned until the replacement publication +retires. An exact terminal drain requires closed recovery, valid tickets, a +covering kick stamp, a fully consumed item ring and matching command accounting. +After that witness, predecessor ownership can retire even if the public fence +failed. An acknowledged context kill must settle its commands before the same +cleanup. An unpublished predecessor remains available for close rollback. +Collector backpressure retains the command and predecessor for another pass. + +Off-lock root reclamation requests another device pass so idle bindings can +release their exited contexts and QoS leases. A closing queue records its idle +time when its last ordinary or spared command retires, allowing warm-pool +expiry to run even if close preceded retirement. + +These transitions do not make an error fence into a lifetime witness, cancel +an installed QID, relax ray-traversal kill exclusions, or release a published +render kill hold. Unwitnessed work remains retained. + +Tests +----- + +GPU KUnit covers predecessor cleanup, finite allocator rollback and repeated +closed, quarantined render leases after an explicit drain witness. The host +``tools/asahi/g17-capacity-fault-tests.py`` compiles the production compute close, +drain, kill, reclaim and recycle methods with mock MMIO. It exercises 512 fault/close and 512 acknowledged-kill cycles, +incomplete stamp/cursor/recovery witnesses, collector backpressure and rollback. +Thirteen mutation controls cover close/fault/kill/reclaim cleanup, shared +compute retirement and kill guards, pool-release admission, and private-pool +release/borrower/construction witnesses, deferred physical admission, and +logical close during off-lock construction. The shared-pool host adapter repeats +512 four-client cycles and keeps destruction outside the device lock. +These tests do not simulate firmware or establish hardware recovery behavior. + +Render admission uses one monotonic absolute deadline across physical identity +retries and the inner pair wait. Hazard/quarantine observations cannot renew +that deadline. Pair waits check packet cancellation, logical close and device +loss before acquiring a lease, and sleep at most 10 ms (rounded to a jiffy) so +cancellation requires no release notification. Timeout/cancellation removes the +waiter's footprint without releasing any active or quarantined lease. + +``g17-admission-deadline-tests.py`` compiles the production wait loop with a +controlled clock and condition variable. It covers all-pairs-busy/no-progress, +unrelated wake storms that report unchanged jiffy budgets, cancellation without +notifications, signal cleanup and an already-expired outer deadline. Native +``asahi_g17_admission_deadline`` KUnit covers real timed waits and cancellation +as well as absolute-deadline arithmetic. + +Compute admission starts one monotonic deadline after dependencies become ready. +The same packet admission context crosses binding-loss retries, shared aperture +and scheduler construction, pool growth, physical-capacity reclamation and the +preparation gate. Every retry checks packet cancellation, logical close, VM +status and device loss. Shared-constructor and closed-gate condition waits use +at most 10 ms slices (rounded to a jiffy); wake reports never reset the deadline. +An already-spent budget refuses even immediately available capacity. Expiry +returns EBUSY and cancellation returns ECANCELED before a new publication. +These are cooperative wait bounds, not preemption of synchronous allocation or +mutex holders. Post-allocation checks reject installation after expiry. + +Growth drops its outer preparation lease before re-entry. Failed growth keeps +its physical token through abort, drops tentative mappings before returning an +unpublished QID, and retains installed firmware-owned backing until its existing +witness. Constructor failure clears its builder flag even if the packet expired +while the off-lock allocation was running. + +``asahi_g17_compute_deadline`` KUnit exercises real preparation-gate timeouts, +cancellation without events, spent budgets and recovery/re-entry churn. +``g17-compute-deadline-tests.py`` executes production ready-packet admission, +its inner witness wait and gate entry with a controlled clock, including the +repeated shared-binding theft interleaving. ``g17-compute-constructor-tests.py`` +executes both shared constructors. ``g17-compute-growth-deadline-tests.py`` +executes the complete growth wait and abort. Their deliberate mutations must +fail at specific deadline, cancellation or lifetime assertions. The pool-release +adapter has only reachable mutation controls and rejects unchanged mutations. +Firmware/MMIO and allocation boundaries in the host adapters remain mocks. diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea0..c4b07c603fb82 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -1119,3 +1119,51 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } + +impl BufferIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", + self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, + self.entries.iter().filter(|entry| entry.owner).count(), + self.entries.iter().filter(|entry| entry.references != 0).count()) + } +} +impl MetricsIds { + pub(crate) fn used(&self) -> u32 { + self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() + - (128 - fw::METRICS_SLOTS as u32) + } +} + +#[kunit_tests(asahi_g17_buffer_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_build_and_fault_retirement_return_parameter_buffers() { + let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); + for _ in 0..BUFFER_IDS { + clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); + } + assert_eq!(ids.allocate_render(), Err(ENOSPC)); + for (index, id) in clients.into_iter().enumerate() { + if index % 2 == 0 { + // Failed constructor never installed the graph owner. + ids.release(id).unwrap(); + } else { + ids.reserve_owner(id).unwrap(); + ids.release(id).unwrap(); + ids.acquire_owner(id).unwrap(); + // Close cannot return a still-owned failed command. + ids.unreserve_owner(id).unwrap(); + assert!(!ids.entries[id as usize].free()); + ids.release(id).unwrap(); + } + } + assert!(ids.entries.iter().all(|entry| entry.free())); + assert_eq!(ids.returned_count, BUFFER_IDS); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs new file mode 100644 index 0000000000000..db2561d4d062f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Name capacity refusals at their source, before a common submit or fence +//! failure handler loses their provenance. Counters are device-lifetime +//! independent diagnostics, never admission or ownership state. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Site { + ComputeReserve, + PhysicalQids, + RenderSlots, + LogicalQueues, + ComputeSlots, + QosIds, + ComputeItems, + RetiredRenderBindings, + RenderPoolIds, + RenderWorkSlots, + ParameterBuffers, + MetricsIds, + LowerVa, + UpperVa, + AliasVa, +} + +const SITE_COUNT: usize = Site::AliasVa as usize + 1; +static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; + +/// First and power-of-two refusals per site keep a pressure storm bounded in +/// the log. This does not change whether the caller retries or rejects work. +#[track_caller] +pub(super) fn refused(site: Site) -> Error { + let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); + if count.is_power_of_two() { + let at = core::panic::Location::caller(); + pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); + } + ENOSPC +} + +/// Read-only diagnostics hold the live device session while sampling its pools. +/// They never wake firmware, reclaim a resource, or change admission state. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), + u8::from(self.0.crashed.load(Ordering::Acquire)))?; + let uat = { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1")?; + firmware.queues.write_capacity(f)?; + firmware.pool.write_capacity(f)?; + writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; + firmware.uat.clone() + }; + // Mapping destruction can issue firmware flushes while holding VM + // locks. Never take a VM/TTBAT lock with the device mutex held here. + // The retained UAT owns valid allocator metadata even during shutdown. + writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let mut upper = uat.geometry().kernel_range(); + upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + for (name, vm, range) in [ + ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), + ("dynamic_upper_bytes", uat.kernel_vm(), upper), + ] { + match vm.capacity(range.clone()) { + Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, + Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, + } + } + for (site, count) in [ + Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, + Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, + Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, + Site::LowerVa, Site::UpperVa, Site::AliasVa, + ].iter().zip(REFUSALS.iter()) { + writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44bee..fe874992cffeb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; Ok(QosLease { @@ -387,7 +387,8 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context.work.lock().render.get(pair).ok_or_else(|| + super::capacity::refused(super::capacity::Site::RenderWorkSlots)) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -406,3 +407,37 @@ impl Drop for WorkStateLease { } } } + +impl QosIds { + pub(crate) fn used(&self) -> u32 { + (*self.used.lock() & !3u128).count_ones() + } +} + +#[kunit_tests(asahi_g17_qos_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_client_construction_returns_every_qos_id() { + let ids = QosIds::new().unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); + for _ in 0..126 { + clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 126); + assert!(matches!(ids.acquire(), Err(ENOSPC))); + drop(clients); + assert_eq!(ids.used(), 0); + assert_eq!(*ids.used.lock(), 3); + } + } +} + +impl Context { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { + writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", + scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 6c48367e2d686..595c7154d42ed 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -69,3 +69,12 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } + +impl WorkStates { + pub(super) fn used(&self) -> usize { + self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() + } +} +impl RenderSlots { + pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 6b4522cea9a15..49544300531a8 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) } } impl Drop for RenderId { @@ -757,3 +757,30 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } + +impl RenderIds { + pub(crate) fn used(&self) -> u32 { + self.used[0].load(Ordering::Acquire).count_ones() + + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() + } +} + +#[kunit_tests(asahi_g17_render_id_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn rejected_pool_construction_returns_all_render_ids() { + let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); + for _ in 0..127 { + pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 127); + assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); + drop(pools); + assert_eq!(ids.used(), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 454fd5d74bdbc..083cfe3d1ce29 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -242,7 +242,8 @@ pub(crate) trait Backend: Send + Sync { None } - /// Lazily create or rebind this logical queue's physical compute graph. + /// Validate compute sizing without reserving a physical channel. Queued + /// commands may wait on a host signal submitted after this ioctl returns. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index e2a9bb643b892..a2f8094dfb996 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -18,7 +18,15 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; /// Minimum render-channel capacity protected from compute-only allocation. -const COMPUTE_RENDER_RESERVE: usize = 16; +// Protect one tiling/fragment pair for each logical execution context, plus +// the two unpublished bootstrap reservations. The remaining ten compute +// channels are shared after exact retirement, rather than pinned per client. +const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +const _: () = { + assert!(COMPUTE_CHANNELS >= 4); + assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); +}; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -133,7 +141,7 @@ impl QueueIds { if QID_COUNT - self.leased + replacement <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } } let index = preferred @@ -141,7 +149,7 @@ impl QueueIds { .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) - .ok_or(ENOSPC)?; + .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { qid: index as u8, owner, @@ -200,6 +208,82 @@ impl QueueIds { } } +#[kunit_tests(asahi_g17_qid_accounting)] +mod accounting_tests { + use super::*; + + fn table() -> KBox { + KBox::init(QueueIds::new(), GFP_KERNEL).unwrap() + } + + #[test] + fn render_identities_satisfy_the_compute_reserve() { + let mut ids = table(); + for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + assert_eq!(ids.leased, QID_COUNT); + } + + #[test] + fn compute_cannot_consume_the_render_minimum() { + let mut ids = table(); + for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + ids.cancel_unpublished(tiling).unwrap(); + assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + } + + #[test] + fn initial_reservations_yield_but_published_ids_do_not() { + let mut ids = table(); + ids.reserve_initial_render().unwrap(); + for owner in 3..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); + let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); + assert_ne!(first.qid(), second.qid()); + assert!(ids.initial_render.iter().all(Option::is_none)); + assert_eq!(ids.leased, QID_COUNT); + ids.publish(first).unwrap(); + assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT); + ids.cancel_unpublished(second).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT - 1); + } + + #[test] + fn failed_second_render_id_returns_only_the_unpublished_first() { + let mut ids = table(); + for owner in 1..QID_COUNT as u64 { + let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + ids.publish(id).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); + ids.cancel_unpublished(first).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); + assert_eq!(replacement.qid(), first.qid()); + assert_ne!(replacement, first); + assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -413,3 +497,20 @@ impl Queue { }) } } + +impl QueueIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let mut engines = [0usize; 3]; + let mut published = 0; + for entry in self.entries.iter().flatten() { + engines[match entry.id.engine { + DataMaster::Tiling => 0, + DataMaster::Fragment => 1, + DataMaster::Compute => 2, + }] += 1; + published += usize::from(entry.published); + } + writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", + self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9049f12ef6608..6e0295e721d42 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -33,6 +33,7 @@ mod ids; pub(crate) mod hazard; mod initdata; mod job; +mod capacity; mod kick; mod object; mod power; @@ -740,12 +741,20 @@ impl Shared { } fn drain_reclaims(self: &Arc) { + let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); + reclaimed = true; + } + if reclaimed { + // Context release returns its execution root off-lock. The next + // device pass can now detach idle compute aliases and their QoS + // leases, without depending on a new client's submission/event. + self.queue_events(); } } @@ -824,6 +833,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _capacity: Pin>>, } impl Gpu { @@ -882,6 +892,11 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", capacity::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -890,6 +905,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + _capacity: capacity, }, GFP_KERNEL, )?; @@ -1157,9 +1173,19 @@ impl gpu::Gpu for Gpu { let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; let policy = scheduling_policy(priority)?; + let admission_started = now_ns(); let (backend, slot) = loop { match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, + Err(error) if error == ENOSPC => { + // Closed logical queues can still own execution/QoS IDs + // until the scheduler release is consumed and reclaimed. + // Backend construction has rolled back all temporary leases. + if now_ns().saturating_sub(admission_started) + >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } result => break result?, } }; diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index a4f06a999eddc..f1b22ad5d7331 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,7 +78,17 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + Placement::In(range, align) => { + let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start + && range.end <= crate::hw::t8140::dynamic::LOWER.end { + super::capacity::Site::LowerVa + } else { + super::capacity::Site::UpperVa + }; + this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + if error == ENOSPC { super::capacity::refused(site) } else { error } + })? + } }; this.mapping = Some(mapping); Ok(this) @@ -113,6 +123,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -159,6 +172,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } fn alias_in_lowest( @@ -373,15 +389,17 @@ impl Allocator<'_> { loop { // Keep ordinary best-fit placement on the uncontended path. A // collision starts a lowest-address search over the whole arena. - let mapping = if searching { + let mapping = (if searching { object.alias_in_lowest( self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, - )? + ) } else { object.obj.map_into_range( self.uat.kernel_lower_vm(), range.clone(), align, prot, true, - )? - }; + ) + }).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) + } else { error })?; let address = mapping.iova(); let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; // Partial aliases are destroyed before releasing the canonical @@ -409,7 +427,9 @@ impl Allocator<'_> { // rather than remapping backing at every intervening page. let candidate = object.alias_in_lowest( aliases[index], start..range.end, align, prot, 0, - )?; + ).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) + } else { error })?; range.start = candidate.iova(); drop(candidate); searching = true; @@ -654,3 +674,16 @@ impl Pool { } } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let inner = self.inner.lock(); + for family in 0..POOL_FAMILIES { + let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; + writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, + inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), + entries.iter().flatten().filter(|entry| entry.poisoned).count())?; + } + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index b10b81c2d8e14..b00a5bb6b6191 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -20,6 +20,34 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; +/// One ready packet's monotonic compute-admission budget. The callback checks +/// packet cancellation, logical close, VM status and device loss even when no +/// capacity/recovery event arrives. Wakes and nested constructors never renew it. +pub(super) struct Admission<'a> { + deadline: u64, + check: &'a dyn Fn() -> Result, +} + +impl<'a> Admission<'a> { + pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { + Self { deadline, check } + } + + pub(super) fn check(&self) -> Result { + (self.check)()?; + if super::now_ns() >= self.deadline { return Err(EBUSY); } + Ok(()) + } + + /// Ignore condvar remaining-time reports: only the monotonic deadline counts. + pub(super) fn slice_ms(&self) -> Result { + self.check()?; + let remaining = self.deadline.saturating_sub(super::now_ns()); + if remaining == 0 { return Err(EBUSY); } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) + } +} + #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -92,6 +120,27 @@ impl Gate { } } + /// Packet admission may wait for recovery only within its original budget. + /// Called with no device mutex or existing preparation lease. + pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let _ = self.changed.wait_timeout( + &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); + } + } + fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -203,3 +252,77 @@ mod growth_gate_tests { drop(growth); } } + +#[kunit_tests(asahi_g17_compute_deadline)] +mod compute_deadline_tests { + use super::*; + + #[test] + fn spent_budget_refuses_even_an_open_gate() { + let gate = Gate::new().unwrap(); + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn closed_gate_expires_without_any_recovery_wake() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(super::super::now_ns() - start < 500_000_000); + gate.wait_drained(); + assert!(gate.reopen()); + } + + #[test] + fn cancellation_without_capacity_or_recovery_events() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || { + if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }; + let admission = Admission::new(start + 2_000_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn recovery_and_binding_retry_churn_cannot_renew_budget() { + let gate = Gate::new().unwrap(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + loop { + // Re-entry models a channel taken between admission/publication. + // Recovery joins each dropped lease, then wakes the same budget. + gate.close(); + gate.changed.notify_all(); + gate.wait_drained(); + assert!(gate.reopen()); + match gate.enter_until(&admission) { + Ok(lease) => drop(lease), + Err(error) => { assert_eq!(error, EBUSY); break; } + } + } + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn packet_status_and_cancel_precede_available_capacity() { + let gate = Gate::new().unwrap(); + for error in [ECANCELED, EIO] { + let check = || Err(error); + let admission = Admission::new(u64::MAX, &check); + assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 6e99160729034..0165a00568118 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -47,6 +47,16 @@ struct InFlight { const QUARANTINE_WAIT_MS: u32 = 1000; +/// A wake never renews admission's budget. Short observations also notice a +/// packet cancellation that has no corresponding pair-release notification. +fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { + let remaining = deadline_ns.saturating_sub(now_ns); + if remaining == 0 { + return Err(EBUSY); + } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) +} + struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], @@ -92,7 +102,7 @@ impl State { & Self::MASK == 0 => { - Err(ENOSPC) + Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) } _ => Ok(()), } @@ -366,7 +376,7 @@ impl Pool { let mut count = self.queues.load(Ordering::Acquire); loop { if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( count, @@ -383,16 +393,22 @@ impl Pool { self: &Arc, owner: u64, request: Option>, + deadline_ns: u64, + check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let started = crate::g17::now_ns(); + let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); + let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); let mut expired = false; let mut ticket = None; let result = (|| loop { + check()?; + let now = crate::g17::now_ns(); + let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -404,20 +420,19 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { + if now >= quarantine_deadline { return Err(EIO); } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; + let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { + return Err(ERESTARTSYS); + } continue; } state.refusal(owner)?; + expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -443,18 +458,17 @@ impl Pool { } } } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { + } + // Keep the established hazard wait/fallback rule, but cap every + // sleep with the caller's original deadline and cancellation poll. + let slice = if plan != Plan::Any { + remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) + } else { + remaining_ms + }; + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { return Err(ERESTARTSYS); } })(); @@ -544,6 +558,75 @@ impl Pool { } } +#[kunit_tests(asahi_g17_admission_accounting)] +mod accounting_tests { + use super::*; + + #[test] + fn every_logical_owner_fits_without_speculative_spares() { + let mut state = State::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot = state.acquire(owner).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); + state.release(slot); + } + assert_eq!(state.active, 0); + assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); + } + + #[test] + fn close_does_not_reuse_an_active_lease() { + let mut state = State::new(); + let slot = state.acquire(1).unwrap(); + state.released |= 1u64 << slot; + assert!(!state.reassign_unused_spare(slot, Some(1), None)); + state.release(slot); + assert!(state.reassign_unused_spare(slot, Some(1), None)); + assert_eq!(state.owners[usize::from(slot)], 0); + assert_eq!(state.refs[usize::from(slot)], 0); + assert_eq!(state.active & (1u64 << slot), 0); + } + + #[test] + fn warming_quarantine_and_exit_hold_exclude_reassignment() { + let mut state = State::new(); + let primary = state.acquire(1).unwrap(); + // A retained legacy sibling still obeys all quarantine/kill guards. + state.owners[2] = 1; + let spare = state.borrow_warm_sibling(1, primary).unwrap(); + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.release(spare); + state.quarantined |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.quarantined = 0; + state.kill_hold |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.kill_hold = 0; + assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); + assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + state.release(primary); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + } + + #[test] + fn logical_queue_count_returns_on_drop() { + let pool = Pool::new().unwrap(); + let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..MAX_EXECUTION_CONTEXTS { + leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); + assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); + drop(leases.pop()); + let replacement = pool.reserve_queue().unwrap(); + drop(leases); + assert_eq!(pool.queues.load(Ordering::Acquire), 1); + drop(replacement); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } +} + /// Counted once per userspace logical queue, independently of installed graphs. pub(crate) struct QueueSlot { pool: Arc, @@ -642,3 +725,147 @@ impl Pool { self.released.notify_all(); } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS)?; + let state = self.state.lock(); + let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { + bits | if *owner != 0 { 1u64 << slot } else { 0 } + }) & State::MASK; + let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; + writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", + used.count_ones(), State::MASK.count_ones(), retained.count_ones(), + (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), + (state.kill_hold & State::MASK).count_ones()) + } +} + +#[kunit_tests(asahi_g17_fault_admission)] +mod fault_capacity_tests { + use super::*; + + #[test] + fn closed_faulting_clients_return_leases_after_drain() { + let pool = Pool::new().unwrap(); + for owner in 1..=512 { + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + lease.quarantine(); + pool.release_owner(owner); + let bit = 1u64 << lease.slot(); + assert_ne!(pool.state.lock().active & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + // The runtime, not an error fence, supplies this exact drain witness. + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.active, 0); + assert_eq!(state.quarantined, 0); + assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } + } + + #[test] + fn published_exit_kill_holds_survive_close_and_drain() { + let pool = Pool::new().unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); + // A failed (unpublished) control may roll back its hold. Published kills + // keep the existing processor-stop rule; this test must not weaken it. + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().kill_hold, 0); + } +} + +impl Pool { + pub(crate) fn owner_active(&self, owner: u64) -> bool { + let state = self.state.lock(); + state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) + } +} + +#[kunit_tests(asahi_g17_shared_render_depth)] +mod shared_depth_tests { + use super::*; + #[test] + fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { + let mut state = State::new(); + let first = state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } + assert_eq!(state.acquire(1), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + state.release(first); + assert_eq!(state.acquire(1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_admission_deadline)] +mod deadline_tests { + use super::*; + + #[test] + fn all_pairs_busy_no_progress_expires_without_release() { + let pool = Pool::new().unwrap(); + { + let mut state = pool.state.lock(); + // No pair has room, including the requesting owner's retained pair. + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + assert!(matches!(result, Err(error) if error == EBUSY)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn busy_pair_observes_cancellation_without_a_release_event() { + let pool = Pool::new().unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }); + assert!(matches!(result, Err(error) if error == ECANCELED)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn cancellation_precedes_available_capacity() { + let pool = Pool::new().unwrap(); + assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + Err(error) if error == ECANCELED)); + assert_eq!(pool.state.lock().active, 0); + } + + #[test] + fn unrelated_wake_storm_never_renews_absolute_deadline() { + let deadline = 2_000_000_000; + // Each wake is allowed to report the full jiffy budget remaining. Only + // monotonic time, not that report or a wake count, bounds admission. + for now in (0..deadline).step_by(100_000) { + assert!(wait_slice_ms(deadline, now).unwrap() <= 10); + } + assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); + assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 54ed90c59392f..c17f737cdd507 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -258,7 +258,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); + return Err(EAGAIN); } } Ok(()) @@ -389,11 +389,21 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous { - binding: Option, - context: Arc, +struct Previous> { + binding: Option, + context: C, publication: Option<(u64, u64)>, } +impl Previous { + /// The exact drain witness covers the replacement publication even when + /// its public fence failed. An unpublished replacement still needs rollback. + fn retire_witnessed(previous: &mut Option) { + if previous.as_ref().is_some_and(|p| p.publication.is_some()) { + drop(previous.take()); + } + } +} + /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -479,6 +489,8 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, + /// This binding has never crossed the successor visibility boundary. + unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -535,6 +547,7 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, + unpublished_binding: true, kick, pool, graph, @@ -640,11 +653,13 @@ impl Queue { matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) } - /// An exited owner's finished command waits only for its completion stamp, - /// which `retirement::STAMP_FALLBACK_NS` bounds. Polling retires it without - /// a new GPU event, after which the queue can be reused. + /// A closed owner can still retain its last commands or their completion + /// stamps. Admission may wait for retirement, without treating that close + /// as permission to reuse any firmware-owned backing. pub(crate) fn release_settling(&self) -> bool { - self.owner.is_none() && self.released && self.retirement_polling() + // release_owner keeps the old owner while its last commands retire. + // Waiting only once owner is None misses precisely that interval. + release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -848,6 +863,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { @@ -948,6 +964,10 @@ impl Queue { self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; + // The stamp and fully consumed ring proved the replacement retired. + // Keeping its predecessor only on the fault path prevents recycling + // forever: neither another retirement nor an exit kill can release it. + Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -956,15 +976,9 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } if self.released { self.owner = None; + self.released_at = now_ns(); } } Ok(()) @@ -1046,6 +1060,63 @@ impl Queue { } } +/// Validate the complete unpublished item budget before changing descriptors, +/// work counts or queue producers. Leave one item empty across ring wrap. +fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { + if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); + return Err(EAGAIN); + } + Ok(()) +} + +/// Logical close promises a future healthy handoff, not immediate reuse. +/// Exact retirement still owns the transition. +fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { + released && !quarantined && (active != 0 || retiring) +} + +#[kunit_tests(asahi_g17_compute_admission)] +mod admission_tests { + use super::*; + + #[test] + fn item_refusal_is_retryable_and_preserves_the_empty_slot() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + } + + #[test] + fn bad_item_cursors_are_not_backpressure() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); + assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + } + + #[test] + fn closed_owner_waits_for_its_last_command() { + assert!(release_pending(true, false, 1, false)); + assert!(release_pending(true, false, 0, true)); + assert!(!release_pending(true, false, 0, false)); + } + + #[test] + fn live_or_quarantined_owners_do_not_promise_capacity() { + assert!(!release_pending(false, false, 1, true)); + assert!(!release_pending(true, true, 1, true)); + } +} + impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1126,6 +1197,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1153,6 +1225,13 @@ impl Queue { } self.retirement_ready = true; } + // Rebinding an already detached queue has no Previous. If its + // first command is rejected or cancelled before publication, the + // empty active FIFO is still the original idle witness. No future + // completion can turn retirement_ready back on for this owner. + if self.previous.is_none() { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1317,9 +1396,7 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } + if !self.can_rearm() { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1651,6 +1728,7 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); + item_capacity(self.graph.cursors()?, entry_signal)?; let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1759,16 +1837,6 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); - } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1776,6 +1844,7 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1812,6 +1881,7 @@ impl Queue { Err(error) })?; visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -2015,3 +2085,153 @@ impl Queue { self.terminal.take() } } + +impl Queue { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", + self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), + u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), + u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; + match self.graph.cursors() { + Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + _ => writeln!(f, "items_unavailable=1"), + }?; + writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), + u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + if let Some(binding) = self.binding.as_ref() { + binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + } + if let Some(previous) = self.previous.as_ref() { + previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + } + Ok(()) + } +} + +#[kunit_tests(asahi_g17_fault_ownership)] +mod fault_ownership_tests { + use super::*; + use core::sync::atomic::AtomicUsize; + + struct Lease(Arc); + impl Lease { + fn new(used: &Arc) -> Self { + used.fetch_add(1, Ordering::Relaxed); + Self(used.clone()) + } + } + impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + + // This tests the production predecessor ownership operation without firmware + // storage. Runtime tests must separately establish its drain/kill witness. + fn faulting_clients(publication: Option<(u64, u64)>) { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + for _ in 0..512 { + let mut previous = Some(Previous { + binding: Some(Lease::new(&used)), + context: Lease::new(&used), + publication, + }); + assert_eq!(used.load(Ordering::Relaxed), 2); + Previous::retire_witnessed(&mut previous); + assert_eq!(previous.is_none(), publication.is_some()); + if publication.is_none() { + assert_eq!(used.load(Ordering::Relaxed), 2); + // An unpublished replacement returns its original owner on close. + drop(previous.take()); + } + assert_eq!(used.load(Ordering::Relaxed), 0); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn faulted_published_replacements_return_to_baseline() { + faulting_clients(Some((1, 7))); + } + + #[test] + fn unpublished_replacements_survive_until_rollback() { + faulting_clients(None); + } + + #[test] + fn same_vm_replacement_without_binding_returns_its_context() { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + let mut previous = Some(Previous:: { + binding: None, + context: Lease::new(&used), + publication: Some((9, 3)), + }); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } +} + +impl Queue { + /// Offer an open client's idle physical channel to another ready job. + /// Logical queue/context ownership is unchanged. The existing binding is + /// retained through the successor's exact replacement witness. + pub(crate) fn share_idle(&mut self) -> Result { + if !self.installed || self.owner.is_none() || !self.retirement_ready + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.previous.is_some() || self.pool_phase != PoolPhase::Populated + { + return Ok(false); + } + self.release_owner()?; + Ok(self.reusable()) + } +} + +impl Queue { + pub(crate) fn can_rearm(&self) -> bool { + self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + } +} + +impl Queue { + /// A failed first attempt has no future completion to undo its binding. + /// Return the replaced aliases to the caller for off-lock destruction. The + /// caller validates the current preparation epoch and cancels only a true + /// result's never-published QID; installed rings and identities stay intact. + pub(crate) fn rollback_unpublished( + &mut self, owner: u64, context: &Arc, + ) -> Result<(bool, Option)> { + if self.owner != Some(owner) || !self.unpublished_binding + || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.retire_pending || self.failure_status_pending + || self.pool_phase != PoolPhase::Populated + || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + { + return Ok((false, None)); + } + if !self.installed { return Ok((true, None)); } + let mut retired = None; + if let Some(previous) = self.previous.as_ref() { + self.graph.set_owner(&previous.context)?; + } + if let Some(mut previous) = self.previous.take() { + if let Some(binding) = previous.binding.take() { + retired = self.binding.replace(binding); + } else { + self.binding.as_mut().ok_or(EIO)?.context = previous.context; + } + } + self.unpublished_binding = false; + self.retirement_ready = true; + self.released = true; + self.owner = None; + self.released_at = now_ns(); + Ok((false, retired)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 187fd27f8e78e..d7386e335e167 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -220,6 +220,9 @@ impl Queue { ))?; drop(self.active.pop()); } + // The acknowledged kill and command settlement replace the ordinary + // drain witness, including a failed first publication after rebinding. + super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 26f1a79f6562a..f9201f6391b38 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -130,7 +130,8 @@ impl Queue { } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Admission is checked even when this queue already owns a scheduler. + // Validate every request, including an existing scheduler. Physical + // admission happens after its DRM dependencies, in backend publication. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index b080cbe604055..e0565bf83db83 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,6 +25,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -119,7 +120,8 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -127,6 +129,12 @@ impl Registry { Rebind::Fresh }); } + // Physical channels are a device budget, not a permanent entitlement + // of each open logical queue. Only exact idle retirement can yield one. + for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } let mut vacant = None; for queue in self .compute @@ -135,6 +143,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + admission.check()?; if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -274,16 +283,11 @@ impl Registry { Ok(freed) } - /// A retained queue in a pool-release transaction, or one whose exited - /// owner's last command waits for its completion stamp, temporarily cannot - /// bind. Distinguish those states from permanent physical-identity - /// exhaustion. + /// Builders and released queues waiting for their exact retirement or pool + /// witness temporarily occupy identities. A failed admission must let those + /// transactions finish rather than fail a healthy successor immediately. fn compute_admission_error(&self, error: Error) -> Error { - if error == ENOSPC - && self.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.pool_release_pending() || queue.release_settling()) - { + if error == ENOSPC { EAGAIN } else { error @@ -307,7 +311,8 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or(ENOSPC)?; + .ok_or_else(|| self.compute_admission_error( + crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; let id = self .qids @@ -416,6 +421,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + closed: core::sync::atomic::AtomicBool, } impl Backend { @@ -465,6 +471,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -474,13 +481,15 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, admission: &Admission<'_>) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -499,7 +508,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } }; let alloc = super::object::Allocator { @@ -510,7 +521,9 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -543,10 +556,12 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, + admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -558,14 +573,18 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -601,8 +620,9 @@ impl Backend { firmware.queues.cancel_compute(reservation) } - fn grow_compute(&self, required: usize) -> Result { + fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { use kernel::time::{delay::fsleep, Delta}; + admission.check()?; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -610,23 +630,23 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); let mut growth = None; let result = (|| loop { let result = (|| { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -643,8 +663,7 @@ impl Backend { deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + Err(error) if error == EAGAIN && lease.is_current() => { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } @@ -679,11 +698,11 @@ impl Backend { result } - fn build_compute(&self, required: usize) -> Result { - const POOL_RELEASE_WAIT_NS: u64 = 200_000_000; - let mut pool_release_wait = None; + fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; + self.admission_open()?; loop { - match self.shared_compute() { + match self.shared_compute(admission) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -692,10 +711,12 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -704,9 +725,19 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context, required)? { + self.admission_open()?; + let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { + Err(error) if error == ENOSPC => { + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; + match rebound { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required); } + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { @@ -714,14 +745,9 @@ impl Backend { Err(error) if error == EAGAIN => { // The event worker owns release observation and // drops vacated aliases/backing off this mutex. - let since = *pool_release_wait.get_or_insert(now_ns()); drop(state); drop(lease); - self.shared.queue_events(); - if now_ns().saturating_sub(since) >= POOL_RELEASE_WAIT_NS { - return Err(EBUSY); - } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_compute_capacity(admission)?; continue; } Err(error) if error == ENOSPC => { @@ -748,13 +774,32 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - let ring = compute::DescriptorRing::new(&alloc)?; + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + Ok((pages, compute::DescriptorRing::new(&alloc)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => { + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // A competing ready owner may have taken/rearmed it while + // backing was built off-lock. Leave rejected backing local + // so it is dropped after the device guard, not in rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm()); + if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -770,10 +815,12 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } + self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -781,7 +828,7 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; + self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); @@ -792,12 +839,14 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - firmware.queues.install_compute(reservation, &mut built) + admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) } }; // The old aliases must disappear before a waiting constructor can @@ -813,9 +862,11 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result != Err(EAGAIN) { - return result; + if result == Err(ENOSPC) { + self.wait_compute_capacity(admission)?; + continue; } + if result != Err(EAGAIN) { return result; } } } } @@ -1166,12 +1217,14 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; for _ in 0..SPARED_WAIT_MS { + admission.check()?; let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1183,7 +1236,10 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + match self.build_compute(required, admission) { + Err(error) if error == EAGAIN => continue, + result => return result, + } } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. @@ -1191,6 +1247,7 @@ impl Backend { } // No new command has been accepted. Keep the spared queue and its // retained work intact until a later admission observes its witness. + admission.check()?; Err(EBUSY) } @@ -1202,13 +1259,24 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; - let mut outer_wait = None; + let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let check = || { + if packet.cancel_requested() { return Err(ECANCELED); } + self.admission_open()?; + if packet.completion.status().get() != 0 { return Err(EIO); } + Ok(()) + }; + let admission = Admission::new(deadline, &check); loop { - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + // All DRM dependencies were satisfied before this scheduler callback. + // Recheck on every retry: pressure may yield an idle binding between + // admission and publication without changing this logical queue. + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(&admission)?; packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1216,9 +1284,7 @@ impl Backend { drop(state); continue; } - if packet.cancel_requested() { - return Err(ECANCELED); - } + admission.check()?; if self .shared .crashed @@ -1239,7 +1305,13 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if firmware.queues.compute(self.owner).is_none() { + drop(state); + drop(lease); + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), @@ -1262,11 +1334,6 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. drop(state); @@ -1389,3 +1456,128 @@ impl super::Firmware { } } + +impl Registry { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.qids.write_capacity(f)?; + self.admission.write_capacity(f)?; + self.buffers.write_capacity(f)?; + writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; + writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; + self.render.write_capacity(f)?; + writeln!(f, "compute_slots used={} total={} reusable={} building={}", + self.compute.iter().flatten().count(), self.compute.len(), + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), + self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; + writeln!(f, "compute_channels used={} total={}", + self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.write_capacity(f)?; + } + Ok(()) + } +} + +impl Backend { + pub(super) fn wait_capacity(&self, started: u64) -> Result { + let check = || self.admission_open(); + let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); + self.wait_compute_capacity(&admission) + } + + fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + admission.check()?; + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + // A short reclaim transaction obeys the same session/epoch gate as + // publication. Never keep this lease while sleeping for capacity. + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { + return Ok(()); + } + for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } + // Consumption, not publication or an elapsed timeout, frees backing. + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + if !firmware.render_control_backpressured() { + let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; + if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } + } + Ok(()) + })(); + deferred.finish(); + self.shared.queue_events(); + match result { + Err(error) if error == EAGAIN => {} + result => result?, + } + admission.check()?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } +} + +#[kunit_tests(asahi_g17_shared_capacity)] +mod shared_capacity_tests { + use super::*; + + #[test] + fn all_logical_render_owners_fit_beside_shared_compute() { + let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + // Physical compute channels are device-owned and bounded independently + // of how many logical clients have queued unsignalled work. + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); + // Every accepted logical render owner now has its physical pair; no + // published identity was cancelled or retyped to achieve the fit. + } + + #[test] + fn four_clients_have_capacity_before_any_owner_closes() { + let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); + for client in 0..4u64 { + for queue in 0..9u64 { + let owner = 1 + client * 9 + queue; + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + } + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert_eq!(leases.len(), 36); + } +} + +impl Backend { + /// Logical close is independent of whether an off-lock builder has already + /// installed a physical graph. No constructor can resurrect a closed owner. + fn admission_open(&self) -> Result { + if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } + if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !self.context.is_current() { return Err(EFAULT); } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f..61c495e93a594 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -22,6 +22,36 @@ impl Backend { self.shared.stop_queues(); } + fn rollback_unpublished_compute(&self) -> Result { + // No outer preparation lease survives compute_packet's return. Hold a + // fresh epoch through alias destruction so recovery can join this work. + loop { + let lease = self.shared.preparations.enter(false)?; + let mut retired = None; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { drop(state); drop(lease); continue; } + let mut binding = None; + if let Some(index) = firmware.queues.compute.iter().position(|entry| { + entry.as_ref().and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.owner() == Some(self.owner)) + }) { + let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; + let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? + .rollback_unpublished(self.owner, &self.context)?; + binding = replaced; + if cancel { + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + } + } + drop(state); + drop(binding); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + return Ok(()); + } + } + /// Complete all failure fanout before signalling this packet's unpublished /// error. Visible failures keep the engine's retained owner through a witness. pub(super) fn finish_publication( @@ -36,6 +66,11 @@ impl Backend { } deferred.finish(); if !packet.is_published() { + if packet.engine() == Engine::Compute { + if self.rollback_unpublished_compute().is_err() { + self.stop_after_error(); + } + } packet.completion.fail_unpublished(error); } Err(error) @@ -46,6 +81,9 @@ impl Backend { } fn release_queue(&self) -> Result { + // A builder may own only an off-lock reservation when close arrives. + // Record intent before scanning queues; every later install/rearm checks it. + self.closed.store(true, Ordering::Release); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -75,7 +113,11 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - Backend::ensure_compute(self, scs_pages) + // A reverse timeline chain must be submitted before the host signal. + // Validate here, but do not consume physical capacity behind unsignalled + // DRM dependencies. compute_packet admits it when the job can run. + crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + .map(|_| ()) } fn fail_vm(&self, error: Error) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 30da1b09eedfa..90ca9ed91c63a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -311,7 +311,7 @@ impl Registry { .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) } - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } @@ -926,3 +926,56 @@ impl Registry { true } } + +impl RenderState { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + self.pools.write_capacity(f)?; + for (slot, entry) in self.entries.iter().enumerate() { + if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { + context.write_capacity(f, "render", slot)?; + } + } + Ok(()) + } +} + +impl Registry { + /// Give idle private backing to another ready client under capacity pressure. + /// Keep graph/QID/context ownership; the next use rearms the same pool after + /// its consumed release, exactly as eviction-driven reclaim does. + pub(super) fn reclaim_idle_render_pools( + &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { break; } + let Some(pool) = entry.pool.as_ref() else { continue; }; + // None means this pool never acquired a firmware registration; + // otherwise require its exact final release to have been consumed. + if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) + || self.admission.owner_active(entry.reservation.owner) { continue; } + // A sibling or replacement may be building off-lock before it has + // installed its pool reference. Its logical owner also excludes us. + let mut idle = true; + for other in render.entries.iter().flatten().filter(|other| { + other.reservation.owner == entry.reservation.owner + || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) + }) { + if other.constructing || other.growing || other.preparing.is_some() + || other.pending_error.is_some() || other.pending_timeout { + idle = false; + break; + } + if let Some(pair) = other.pair.as_ref() { + if !pair.idle_for_context(pair.context())? { idle = false; break; } + } + } + if idle { + if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index f7f48e853f225..4713eb5b6b152 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -239,3 +239,11 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } + +impl Pools { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "render_pool_leases used={} total={} retained={}", + self.leases.iter().flatten().filter(|lease| !lease.released()).count(), + POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 13e0e9a9d6d40..411b4f0fd962f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -470,7 +470,7 @@ impl super::super::Backend { if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } firmware.queues.teardown.displace( pair.context(), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index abea0b4c06f29..a27e2222955dd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -35,7 +35,11 @@ impl Backend { let state = self.shared.state.lock(); (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() }; + let started = super::super::now_ns(); + let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); loop { + self.admission_open()?; + if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } if packet.cancel_requested() { return Err(ECANCELED); } @@ -45,7 +49,16 @@ impl Backend { let lease = if let Some(lease) = next.take() { lease } else { - match pool.acquire_render(self.owner, Request::of(packet)) { + match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + // Atomics only: the callback runs under the pool mutex. + if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + Ok(()) + }) { Ok(lease) => lease, Err(error) if error == ENOSPC => { let reclaimed = { @@ -57,7 +70,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - return Err(error); + self.wait_render_capacity(started)?; } drop(reclaimed); continue; @@ -91,7 +104,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - return Err(error); + self.wait_render_capacity(started)?; } drop(reclaimed); } @@ -100,6 +113,13 @@ impl Backend { } } + /// A closing owner may still hold its pair until exact retirement. Wait + /// outside both the device mutex and preparation gate so the event worker + /// and recovery can return capacity. No packet is firmware-visible here. + fn wait_render_capacity(&self, started: u64) -> Result { + self.wait_capacity(started) + } + fn render_with_lease( &self, packet: &Arc, @@ -107,6 +127,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -121,6 +142,7 @@ impl Backend { let mut state_delay = 20i64; let mut takeover_polls = 0; let mut recovery_polls = 0; + let capacity_started = super::super::now_ns(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -157,6 +179,10 @@ impl Backend { } } match self.ensure_render(packet, slot) { + Err(error) if error == ENOSPC => { + self.wait_capacity(capacity_started)?; + continue; + } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { if takeover_polls == 5000 { @@ -169,6 +195,10 @@ impl Backend { result => result?, } match self.prepare_render(slot, packet, deferred) { + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_capacity(capacity_started)?; + continue; + } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f2..41f04be56e28d 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -393,7 +393,9 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + // One physical pair per logical owner leaves room for every execution + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4322c88de0651..6a8afe344a316 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2858,3 +2858,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = lock_vm(&self.gpuvm, None, false)?; + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/tools/asahi/g17-admission-deadline-tests.py b/tools/asahi/g17-admission-deadline-tests.py new file mode 100644 index 0000000000000..5002f70997307 --- /dev/null +++ b/tools/asahi/g17-admission-deadline-tests.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production pool wait against a deterministic clock/condition variable. + +Every pair remains busy. Fake wakes report an unchanged jiffy budget, including +unrelated wake storms. This tests elapsed time and cancellation, not firmware. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + begin = source.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + source = (ROOT / 'queue/admission.rs').read_text() + return r''' +#![allow(dead_code)] +use std::sync::{Arc, Mutex, atomic::{AtomicU64, AtomicUsize, Ordering}}; +type Result = std::result::Result; +const EBUSY:i32=-16;const EIO:i32=-5;const EINVAL:i32=-22; +const ERESTARTSYS:i32=-512;const ECANCELED:i32=-125; +static NOW:AtomicU64=AtomicU64::new(0); +static WAITS:AtomicUsize=AtomicUsize::new(0); +static MODE:AtomicUsize=AtomicUsize::new(0); +mod g17 {pub fn now_ns()->u64 {crate::NOW.load(crate::Ordering::SeqCst)}} +mod kernel {pub mod time {pub fn msecs_to_jiffies(ms:u32)->u64 {u64::from(ms)}}} +mod hazard {pub const WAIT_MS:u32=1000;} +const QUARANTINE_WAIT_MS:u32=1000; +#[derive(Clone,Copy,PartialEq)] enum Plan {Any,Conflict} +#[derive(Clone,Copy)] struct Request<'a>(&'a ()); +struct State {owners:[u64;2],quarantined:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} +impl State { + fn exhausted(&self)->bool {false} + fn refusal(&self,_:u64)->Result {Ok(())} + fn plan(&self,_:u64,_:Option>,_:Option)->Plan {if self.conflict {Plan::Conflict}else{Plan::Any}} + fn pick(&mut self,_:u64,_:Plan)->Option {if self.ready{Some(1)}else{None}} + fn record(&mut self,_:u64,slot:Option,_:u64,_:Request<'_>)->Option {if slot.is_none(){self.waiters+=1;}Some(0)} + fn forget_waiter(&mut self,t:Option){if t.is_some(){self.waiters-=1;}} +} +struct Lock(Mutex); +impl Lock {fn lock(&self)->std::sync::MutexGuard<'_,State>{self.0.lock().unwrap()}} +enum CondVarTimeoutResult {Signal{jiffies:u64},Timeout,Woken{jiffies:u64}} +struct CondVar; +impl CondVar { + fn wait_interruptible_timeout(&self,_:&mut std::sync::MutexGuard<'_,State>,slice:u64)->CondVarTimeoutResult { + assert!(WAITS.fetch_add(1,Ordering::SeqCst)<100_000,"wait did not terminate"); + assert!(slice<=10,"cancellation observation must be bounded"); + match MODE.load(Ordering::SeqCst) { + 0=>{NOW.fetch_add(slice*1_000_000,Ordering::SeqCst);CondVarTimeoutResult::Timeout}, + 1=>{NOW.fetch_add(100_000,Ordering::SeqCst);CondVarTimeoutResult::Woken{jiffies:u64::MAX}}, + _=>CondVarTimeoutResult::Signal{jiffies:slice}, + } + } +} +struct Pool {state:Lock,released:CondVar} +struct RenderLease {pool:Arc,slot:u8,record:Option} +''' + item(source, 'fn wait_slice_ms(') + '\nimpl Pool {\n' + item(source, 'pub(crate) fn acquire_render(') + r''' +} +fn new_pool(conflict:bool,quarantined:u64)->Arc { + Arc::new(Pool{state:Lock(Mutex::new(State{owners:[0,1],quarantined,next_ticket:1,waiters:0,conflict,ready:false})),released:CondVar}) +} +fn reset(mode:usize){NOW.store(0,Ordering::SeqCst);WAITS.store(0,Ordering::SeqCst);MODE.store(mode,Ordering::SeqCst);} +fn main(){ + for mode in 0..2 { + for conflict in [false,true] { + reset(mode);let pool=new_pool(conflict,0); + let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())); + assert!(matches!(result,Err(EBUSY)),"busy wait must expire"); + assert!(NOW.load(Ordering::SeqCst)<=2_010_000_000,"absolute deadline extended by wakes"); + assert_eq!(pool.state.lock().waiters,0,"expired waiter must be removed"); + reset(mode);let pool=new_pool(conflict,0); + let result=pool.acquire_render(1,Some(Request(&())),2_000_000_000,||if g17::now_ns()>=5_000_000 {Err(ECANCELED)}else{Ok(())}); + assert!(matches!(result,Err(ECANCELED)),"cancel must interrupt a busy wait"); + assert!(NOW.load(Ordering::SeqCst)<=15_000_000,"cancel observation bound"); + assert_eq!(pool.state.lock().waiters,0,"cancelled waiter must be removed"); + } + reset(mode);let pool=new_pool(false,2); + assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EIO))); + assert!(NOW.load(Ordering::SeqCst)<=1_010_000_000,"quarantine budget extended"); + } + reset(2);let pool=new_pool(true,0); + assert!(matches!(pool.acquire_render(1,Some(Request(&())),2_000_000_000,||Ok(())),Err(ERESTARTSYS))); + assert_eq!(pool.state.lock().waiters,0); + // A previous outer retry already spent the budget. Even available capacity + // cannot restart it or install a lease after cancellation. + reset(0);NOW.store(2_000_000_000,Ordering::SeqCst);let pool=new_pool(false,0);pool.state.lock().ready=true; + assert!(matches!(pool.acquire_render(1,None,2_000_000_000,||Ok(())),Err(EBUSY))); + assert!(matches!(pool.acquire_render(1,None,u64::MAX,||Err(ECANCELED)),Err(ECANCELED))); + assert_eq!(WAITS.load(Ordering::SeqCst),0); + println!("PASS production admission: all-pairs-busy/no-progress and 20,000 unrelated wakes expire at absolute deadline; no-event cancellation bounded; conflict/quarantine/signal cleanup preserved"); +} +''' + + +def main(): + source = fixture() + caller = (ROOT / 'runtime/render/submit.rs').read_text() + assert 'Request::of(packet), deadline_ns, || {' in caller + assert 'packet.cancel_requested() || self.closed.load(Ordering::Acquire)' in caller + cases = [('fixed', source), + ('renew-deadline-on-wake', source.replace('wait_slice_ms(deadline_ns, now)?', 'wait_slice_ms(now.saturating_add(2_000_000_000), now)?')), + ('ignore-cancel', source.replace('check()?;', ''))] + with tempfile.TemporaryDirectory(prefix='g17-admission-deadline-') as tmp: + for name, code in cases: + path = Path(tmp) / (name + '.rs') + path.write_text(code) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=15) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0, name + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-admission-order-tests.py b/tools/asahi/g17-admission-order-tests.py new file mode 100644 index 0000000000000..c9908aca591c6 --- /dev/null +++ b/tools/asahi/g17-admission-order-tests.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""The production ioctl sizing callback must not allocate physical channels. + +A mock physical allocator records demand while reverse timeline dependencies +remain unsignalled. Restoring the eager callback must fail this same control. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def method(path, signature): + source = (ROOT / path).read_text() + start = source.index(signature) + begin = source.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def main(): + code = r''' +#![allow(dead_code)] +use std::sync::atomic::{AtomicUsize,AtomicBool,Ordering}; +type Result = std::result::Result; +const ECANCELED:i32=-125;const EIO:i32=-5;const EFAULT:i32=-14; +mod g17 {pub mod fw {pub mod compute_pool { + pub fn required_blocks(cores:u32,pages:u16)->crate::Result { + if cores==0 {Err(-22)} else {Ok(21+usize::from(pages))} + } +}}} +struct Status(i32); +impl Status {fn get(&self)->i32{self.0}} +struct Context {status:Status} +impl Context {fn status(&self)->&Status{&self.status} fn is_current(&self)->bool{true}} +struct Shared {active_cores:u32,crashed:AtomicBool} +struct Backend {shared:Shared,physical:AtomicUsize,closed:AtomicBool,context:Context} +impl Backend { + fn ensure_compute(&self,_:u16)->Result {self.physical.fetch_add(1,Ordering::SeqCst);Ok(())} +''' + method('runtime.rs', 'fn admission_open(') + r''' +} +trait Api {fn ensure_compute(&self,scs_pages:u16)->Result;} +impl Api for Backend { +''' + method('runtime/backend.rs', 'fn ensure_compute(') + r''' +} +fn main(){ + let backend=Backend{shared:Shared{active_cores:10,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; + // Four clients each enqueue nine queues in reverse dependency order. The + // final host signal cannot be submitted if ioctl admission blocks on channels. + for _client in 0..4 {for _queue in (0..9).rev() {::ensure_compute(&backend,0).unwrap();}} + assert_eq!(backend.physical.load(Ordering::SeqCst),0,"unsignalled work must not pin physical capacity"); + Backend::ensure_compute(&backend,0).unwrap(); + assert_eq!(backend.physical.load(Ordering::SeqCst),1); + let invalid=Backend{shared:Shared{active_cores:0,crashed:AtomicBool::new(false)},physical:AtomicUsize::new(0),closed:AtomicBool::new(false),context:Context{status:Status(0)}}; + assert_eq!(::ensure_compute(&invalid,0),Err(-22)); + assert_eq!(invalid.physical.load(Ordering::SeqCst),0); + for _ in 0..512 { + backend.closed.store(true,Ordering::Release); + assert_eq!(backend.admission_open().and_then(|()| Backend::ensure_compute(&backend,0)),Err(ECANCELED),"closed constructor must not install"); + assert_eq!(backend.physical.load(Ordering::SeqCst),1); + } + println!("PASS closed builders cannot install after logical close; production ioctl sizing validates 36 reverse-order submissions without physical admission; invalid sizing consumes no channel"); +} +''' + fixed = 'crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)\n .map(|_| ())' + assert code.count(fixed) == 1 + with tempfile.TemporaryDirectory(prefix='g17-admission-order-') as tmp: + for name, source in [('fixed', code), ('eager-physical-admission', code.replace(fixed, 'Backend::ensure_compute(self, scs_pages)')), ('ignore-logical-close', code.replace('self.closed.load(Ordering::Acquire)', 'false'))]: + path = Path(tmp) / (name + '.rs') + binary = path.with_suffix('') + path.write_text(source) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and ('must not pin physical capacity' if name == 'eager-physical-admission' else 'closed constructor must not install') in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-capacity-fault-tests.py b/tools/asahi/g17-capacity-fault-tests.py new file mode 100644 index 0000000000000..09a70caaaccb4 --- /dev/null +++ b/tools/asahi/g17-capacity-fault-tests.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production compute close/retirement/recycle methods with fake MMIO. + +No firmware behavior is inferred from these controls. The actual cursor/stamp +predicate is compiled too; allocation and asynchronous notification are stubs. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + source = (ROOT / 'queue/compute.rs').read_text() + retirement = (ROOT / 'queue/compute/retirement.rs').read_text() + previous = item(source, 'struct Previous<') + '\n' + item(source, 'impl Previous') + methods = '\n'.join(item(source, sig) for sig in [ + 'pub(crate) fn release_owner(', 'pub(crate) fn reusable(', + 'pub(crate) fn recycle_quarantined_idle(', 'pub(crate) fn recycle_killed(', + 'fn recycle_settled(', 'fn release_witnessed_inner(', 'pub(crate) fn share_idle(', 'pub(crate) fn can_rearm(', 'pub(crate) fn recycled_pending(']) + exit_source = (ROOT / 'queue/compute/exit.rs').read_text() + methods += '\n' + item(exit_source, 'pub(crate) fn settle_exit(').replace('super::Previous', 'Previous').replace('crate::g17::qos::Owner', 'qos::Owner') + drain = item((ROOT / 'mod.rs').read_text(), 'fn drain_reclaims(') + proof = item(retirement, 'pub(crate) const fn stamp_covers(') + '\n' + item(retirement, 'pub(crate) fn quarantined_retired(') + return r''' +#![allow(dead_code)] +use std::sync::{Arc, atomic::{AtomicUsize, AtomicU32, Ordering}}; +type Error = i32; +type Result = std::result::Result; +const EIO: Error = -5; +const EBUSY: Error = -16; +const EINVAL: Error = -22; +const ENODATA: Error = -61; +const ECANCELED: Error = -125; +fn now_ns() -> u64 { 123 } +struct Context { id: u64, live: Arc, killed: AtomicU32 } +impl Context { + fn new(id: u64, live: &Arc) -> Arc { + live.fetch_add(1, Ordering::SeqCst); + Arc::new(Self { id, live: live.clone(), killed: AtomicU32::new(1) }) + } +} +impl Drop for Context { fn drop(&mut self) { self.live.fetch_sub(1, Ordering::SeqCst); } } +struct Binding { context: Arc } +''' + previous + r''' +#[derive(Clone, Copy, PartialEq)] enum PoolPhase { Populated, ReleaseSent(u32), Vacant } +#[derive(Clone, Copy)] struct Ticket { + owner: u64, item_count: u32, descriptor: u64, item_producer: u32, kick: u64, +} +#[derive(Clone, Copy)] struct Cursors { count: u32, producer: u32, consumer: u32 } +''' + proof + r''' +mod retirement { pub(crate) use super::quarantined_retired; } +struct Graph { cursors: Cursors, stamp: u32, fail: bool, resets: usize, owner: u64 } +impl Graph { + fn set_owner(&mut self, context: &Context) -> Result { if self.fail { Err(EIO) } else { self.owner = context.id; Ok(()) } } + fn reset_items(&mut self) -> Result { if self.fail { Err(EIO) } else { self.resets += 1; Ok(()) } } + fn cursors(&self) -> Result { Ok(self.cursors) } + fn stamp(&self) -> Result { Ok(self.stamp) } +} +struct Kick; +impl Kick { fn clear_parent_after_recovery(&mut self) {} } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { Published, Acknowledged, Unproven, Released } +struct Exit { phase: Phase } +impl Exit { fn released(&self) -> bool { self.phase == Phase::Released } } +struct State { killed: u32 } +impl State { fn read(context: &Context) -> Result { Ok(Self { killed: context.killed.load(Ordering::SeqCst) }) } } +struct Completion; +impl Completion { + fn has_firmware_consumer(&self) -> bool { false } + fn compute_words(&self) -> Result<[u32; 2]> { Ok([0, 0]) } + fn replay_witness_words(&self) -> Option<[u32; 2]> { Some([0, 0]) } +} +struct Packet { completion: Arc } +struct Active { packet: Arc, ticket: Option, qos_pending: bool } +struct Fifo { entries: Vec, len: usize } +impl Fifo { + fn new() -> Self { Self { entries: vec![], len: 0 } } + fn iter(&self) -> impl DoubleEndedIterator { self.entries.iter() } + fn iter_mut(&mut self) -> impl Iterator { self.entries.iter_mut() } + fn front(&self) -> Option<&T> { self.entries.first() } + fn back(&self) -> Option<&T> { self.entries.last() } + fn push(&mut self, value:T) -> Result { self.entries.push(value); self.len += 1; Ok(()) } + fn pop(&mut self) -> Option { if self.len == 0 { None } else { self.len -= 1; Some(self.entries.remove(0)) } } +} +mod qos { pub type Owner = u64; } +enum Deferred { Retired(Arc, Result<[u64; 4]>), Release(Arc) } +struct Queue { + owner: Option, installed: bool, active: Fifo, quarantined: bool, + previous: Option, unpublished_binding: bool, released: bool, graph: Graph, binding: Option, + retirement_ready: bool, released_at: u64, growing: bool, pool_phase: PoolPhase, + needs_fresh_backing: bool, exit: Option, replays: Fifo>, + awaiting_witness: bool, quarantine_error: Option, failure_status_pending: bool, + retired_by_teardown: bool, spared_quarantine: bool, spared_deferred: bool, + retire_pending: bool, retirement_proved: bool, replay_retirement: bool, + replay_error: Option, replay_prepared: bool, submitted: u32, kick: Kick, +} +impl Queue { + fn valid_ticket(&self, _: &Active, _: &Ticket) -> Result { Ok(()) } + fn qos_owner(&self) -> u64 { 1 } + fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } +''' + methods + r''' +} +fn idle(live: &Arc) -> Queue { + Queue { + owner: Some(2), installed: true, active: Fifo::new(), quarantined: false, + previous: None, unpublished_binding: false, released: false, + graph: Graph { cursors: Cursors { count: 1280, producer: 4, consumer: 4 }, stamp: 7 << 8, fail: false, resets: 0, owner: 2 }, + binding: Some(Binding { context: Context::new(2, live) }), retirement_ready: false, + released_at: 0, growing: false, pool_phase: PoolPhase::Populated, + needs_fresh_backing: false, exit: None, replays: Fifo::new(), + awaiting_witness: false, quarantine_error: None, failure_status_pending: false, + retired_by_teardown: false, spared_quarantine: false, spared_deferred: false, + retire_pending: false, retirement_proved: false, replay_retirement: false, + replay_error: None, replay_prepared: false, submitted: 0, kick: Kick, + } +} +fn fault(live: &Arc) -> Queue { + let mut queue = idle(live); + let old = Context::new(1, live); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: Some((11, 3)) }); + queue.active.push(Active { packet: Arc::new(Packet { completion: Arc::new(Completion) }), + ticket: Some(Ticket { owner: 2, item_count: 1280, descriptor: 0x1000, item_producer: 4, kick: 7 }), qos_pending: true }).unwrap(); + queue.quarantined = true; + queue.awaiting_witness = true; + queue.submitted = 1; + queue.quarantine_error = Some(EIO); + queue +} +struct Reclaim(Arc); +impl Reclaim { fn finish(self) { self.0.fetch_add(1, Ordering::SeqCst); } } +struct Reclaims(Vec); +impl Reclaims { fn take_one(&mut self) -> Option { self.0.pop() } } +struct Mutex(std::sync::Mutex); +impl Mutex { fn lock(&self) -> std::sync::MutexGuard<'_, T> { self.0.lock().unwrap() } } +struct Shared { reclaim: Mutex, wakes: AtomicUsize } +impl Shared { + fn queue_events(&self) { self.wakes.fetch_add(1, Ordering::SeqCst); } +''' + drain + r''' +} +fn main() { + let finished = Arc::new(AtomicUsize::new(0)); + let shared = Arc::new(Shared { reclaim: Mutex(std::sync::Mutex::new(Reclaims(vec![Reclaim(finished.clone())]))), wakes: AtomicUsize::new(0) }); + shared.drain_reclaims(); + assert_eq!(finished.load(Ordering::SeqCst), 1); + assert_eq!(shared.wakes.load(Ordering::SeqCst), 1); + shared.drain_reclaims(); + assert_eq!(shared.wakes.load(Ordering::SeqCst), 1); + let live = Arc::new(AtomicUsize::new(0)); + for _ in 0..512 { + // Rebound detached queue; lost VM / killed publisher / allocation error + // before the first new command. No future completion exists to free it. + let mut queue = idle(&live); + assert!(!queue.release_owner().unwrap()); + assert!(queue.owner.is_none()); + assert!(queue.reusable()); + drop(queue); + assert_eq!(live.load(Ordering::SeqCst), 0); + + let mut queue = fault(&live); + queue.release_owner().unwrap(); + assert!(!queue.recycle_quarantined_idle().unwrap()); + for control in 0..3 { + queue.graph.stamp = if control == 0 { 6 << 8 } else { 7 << 8 }; + queue.graph.cursors.consumer = if control == 1 { 3 } else { 4 }; + queue.release_witnessed_inner(if control == 2 { 1 } else { 0 }, false, |_| Ok(()), &mut |_| Ok(())).unwrap(); + assert_eq!(queue.active.len, 1); + assert!(queue.previous.is_some()); + assert!(!queue.recycle_quarantined_idle().unwrap()); + } + queue.graph.stamp = 7 << 8; + queue.graph.cursors.consumer = 4; + // Collector pressure must retain exact ownership for the next pass. + assert_eq!(queue.release_witnessed_inner(0, false, |_| Ok(()), &mut |_| Err(EBUSY)), Err(EBUSY)); + assert!(queue.previous.is_some()); + queue.release_witnessed_inner(0, false, |_| Ok(()), &mut |_| Ok(())).unwrap(); + assert_eq!(queue.active.len, 0); + assert!(queue.previous.is_none()); + assert_eq!(live.load(Ordering::SeqCst), 1); + assert!(queue.recycle_quarantined_idle().unwrap()); + assert_eq!(queue.graph.resets, 1); + assert!(!queue.reusable()); // old killed/faulted backing must be released + queue.pool_phase = PoolPhase::Vacant; // fake control-consumption boundary + assert!(queue.reusable()); + assert!(queue.can_rearm()); + drop(queue); + assert_eq!(live.load(Ordering::SeqCst), 0); + } + for _ in 0..512 { + let mut queue = fault(&live); + queue.release_owner().unwrap(); + queue.exit = Some(Exit { phase: Phase::Acknowledged }); + queue.context().unwrap().killed.store(0, Ordering::SeqCst); + queue.settle_exit(true, &mut |_| Ok(()), &mut |_| Ok(())).unwrap(); + assert!(queue.previous.is_some()); + assert!(queue.exit.as_ref().unwrap().phase == Phase::Unproven); + assert_eq!(queue.active.len, 1); + queue.context().unwrap().killed.store(1, Ordering::SeqCst); + queue.settle_exit(false, &mut |_| Ok(()), &mut |_| Ok(())).unwrap(); + assert_eq!(queue.active.len, 1); + queue.settle_exit(true, &mut |_| Ok(()), &mut |_| Ok(())).unwrap(); + assert!(queue.previous.is_none()); + assert!(queue.recycle_killed().unwrap()); + assert_eq!(queue.graph.resets, 1); + assert!(!queue.reusable()); + queue.pool_phase = PoolPhase::Vacant; + assert!(queue.reusable()); + drop(queue); + assert_eq!(live.load(Ordering::SeqCst), 0); + } + for _ in 0..512 { + let mut queue = idle(&live); + assert!(!queue.share_idle().unwrap(), "unretired binding cannot be shared"); + assert_eq!(queue.owner, Some(2)); + queue.retirement_ready = true; + queue.exit = Some(Exit { phase: Phase::Acknowledged }); + assert!(!queue.share_idle().unwrap(), "issued kill cannot yield a channel"); + assert_eq!(queue.owner, Some(2)); + queue.exit = None; + assert!(queue.share_idle().unwrap()); + assert!(queue.reusable()); + assert_eq!(queue.graph.resets, 0, "healthy sharing keeps the installed ring"); + assert!(!queue.needs_fresh_backing); + assert_eq!(live.load(Ordering::SeqCst), 1, "predecessor remains retained"); + assert!(!queue.share_idle().unwrap()); + drop(queue); + assert_eq!(live.load(Ordering::SeqCst), 0); + } + let mut queue = idle(&live); + let old = Context::new(1, &live); + queue.previous = Some(Previous { binding: Some(Binding { context: old.clone() }), context: old, publication: None }); + queue.release_owner().unwrap(); + assert_eq!(queue.graph.owner, 1); + assert!(queue.previous.is_none()); + assert!(queue.reusable()); + drop(queue); + assert_eq!(live.load(Ordering::SeqCst), 0); + println!("PASS 512 fault/close and 512 acknowledged-kill cycles: baseline resources restored; stale stamp, cursor, recovery and collector controls preserve ownership; unpublished rollback preserved"); +} +''' + + +def main(): + source = fixture() + idle_fix = '''if self.previous.is_none() { + self.retirement_ready = true; + }''' + fault_fix = 'Previous::retire_witnessed(&mut self.previous);' + assert source.count(idle_fix) == 1 + assert source.count(fault_fix) == 2 + controls = [('fixed', source, True), + ('missing-idle-close', source.replace(idle_fix, ''), False), + ('missing-fault-release', source.replace(fault_fix, '', 1), False), + ('missing-kill-release', source.replace('Previous::retire_witnessed(&mut self.previous);\n self.exit', 'self.exit'), False), + ('missing-reclaim-wake', source.replace('self.queue_events();', ''), False), + ('share-before-retirement', source.replace('|| !self.retirement_ready', ''), False), + ('share-during-kill', source.replace('|| self.exit.is_some() || self.active.len', '|| self.active.len'), False)] + with tempfile.TemporaryDirectory(prefix='g17-capacity-') as tmp: + for name, rust, passes in controls: + path = Path(tmp) / (name + '.rs') + binary = Path(tmp) / name + path.write_text(rust) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], text=True, capture_output=True) + if passes: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0, name + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-capacity-watch.sh b/tools/asahi/g17-capacity-watch.sh new file mode 100644 index 0000000000000..5438413b2ec21 --- /dev/null +++ b/tools/asahi/g17-capacity-watch.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Run inside an already authorized test session. Does not mount debugfs. +set -eu +capacity_file=${1:-/sys/kernel/debug/asahi-g17/capacity} +interval=${2:-5} +case "$interval" in ''|*[!0-9]*|0) echo 'interval must be positive integer seconds' >&2; exit 2;; esac +test -r "$capacity_file" +while :; do + printf 'sample_begin utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + cat "$capacity_file" + printf 'sample_end\n' + sleep "$interval" +done diff --git a/tools/asahi/g17-compute-constructor-tests.py b/tools/asahi/g17-compute-constructor-tests.py new file mode 100644 index 0000000000000..f4fdcf2c68222 --- /dev/null +++ b/tools/asahi/g17-compute-constructor-tests.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Execute both production shared-constructor waits and flag cleanup. + +Allocations, mappings, device locks and firmware are mocked. Real method bodies +must check the original admission after every wake and after off-lock creation. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(text, signature): + start = text.index(signature) + begin = text.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + preparation = (ROOT / 'preparation.rs').read_text() + admission = (item(preparation, "pub(super) struct Admission<'a>") + '\n' + + item(preparation, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()') + methods = '\n'.join(item(runtime, sig) for sig in [' fn shared_compute(', ' fn compute_scheduler(']) + methods = methods.replace('super::object::Allocator', 'Allocator').replace('super::preparation::Lease', 'Lease').replace('super::initdata::', '') + return r''' +#![allow(dead_code)] +use std::sync::{Mutex,MutexGuard,atomic::{AtomicBool,AtomicU64,Ordering}}; +type Result = std::result::Result; +const EIO:i32=-5;const EAGAIN:i32=-11;const ENODEV:i32=-19;const EBUSY:i32=-16;const ECANCELED:i32=-125; +static NOW:AtomicU64=AtomicU64::new(0);static CANCEL:AtomicBool=AtomicBool::new(false);static ALLOCATE:AtomicBool=AtomicBool::new(false); +fn now_ns()->u64{NOW.load(Ordering::SeqCst)} +mod kernel{pub mod time{pub fn msecs_to_jiffies(ms:u32)->u32{ms}}} +''' + admission + r''' +struct Gate;struct Lease; +impl Gate{fn enter_until(&self,a:&Admission<'_>)->Result{a.check()?;Ok(Lease)}} +impl Lease{fn is_current(&self)->bool{true}} +struct State(Mutex>>); +impl State{fn lock(&self)->MutexGuard<'_,Option>>{self.0.lock().unwrap()}} +struct Changed; +impl Changed{ + fn wait_timeout(&self,_:&mut MutexGuard<'_,T>,_:u32){NOW.fetch_add(1_000_000,Ordering::SeqCst);assert!(now_ns()<100_000_000,"constructor wait exceeded original budget");} + fn notify_all(&self){} +} +struct Shared{preparations:Gate,state:State,changed:Changed,crashed:AtomicBool} +impl Shared{fn drm(&self)->Result<()>{Ok(())}} +struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Backend{shared:Shared,context:Context} +struct Allocator<'a>{dev:&'a (),uat:&'a ()} +struct ComputeShared;struct ComputeScheduler; +fn allocate(){if ALLOCATE.load(Ordering::SeqCst){NOW.fetch_add(25_000_000,Ordering::SeqCst);}} +impl ComputeShared{fn new(_:&Allocator<'_>)->Result{allocate();Ok(Self)}} +impl ComputeScheduler{fn new(_:&Allocator<'_>)->Result{allocate();Ok(Self)}} +struct Init{shared:bool,scheduler:bool} +impl Init{ + fn has_compute_shared(&self)->bool{self.shared} + fn has_compute_scheduler(&self)->bool{self.scheduler} + fn cache_compute_vm(&self,_:&())->Result{Ok(())} + fn install_compute_shared(&mut self,_:ComputeShared)->Result{self.shared=true;Ok(())} + fn install_compute_scheduler(&mut self,_:ComputeScheduler)->Result{self.scheduler=true;Ok(())} +} +struct Queues{compute_building:bool,compute_scheduler_building:bool} +struct Firmware{init:Init,uat:(),queues:Queues} +impl Backend{ +''' + methods + r''' +} +fn main(){ + for scheduler in [false,true]{for cancel in [false,true]{for building in [false,true]{ + NOW.store(0,Ordering::SeqCst);CANCEL.store(cancel,Ordering::SeqCst);ALLOCATE.store(!building,Ordering::SeqCst); + let b=Backend{context:Context,shared:Shared{preparations:Gate,changed:Changed,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:Init{shared:false,scheduler:false},queues:Queues{compute_building:building,compute_scheduler_building:building}}))))}}; + let check=||{if CANCEL.load(Ordering::SeqCst)&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; + let admission=Admission::new(20_000_000,&check); + let alloc=Allocator{dev:&(),uat:&()}; + let r=if scheduler{b.compute_scheduler(&alloc,&Lease,&admission)}else{b.shared_compute(&admission)}; + assert_eq!(r,Err(if cancel{ECANCELED}else{EBUSY}),"constructor must propagate deadline/cancel"); + assert_eq!(now_ns(),if building{if cancel{10_000_000}else{20_000_000}}else{25_000_000}); + let state=b.shared.state.lock();let fw=state.as_ref().unwrap(); + assert!(!fw.init.shared&&!fw.init.scheduler,"expired constructor must not install"); + if !building{assert!(!fw.queues.compute_building&&!fw.queues.compute_scheduler_building,"failed constructor must clear builder flag");} + }}} + println!("PASS 8 production shared/scheduler constructor waits: expiry, cancellation, off-lock-build expiry, no installation and flag cleanup"); +} +''' + + +def main(): + source = fixture() + mutants = [ + ('fixed', source, ''), + ('ignore-constructor-wakes', source.replace(' admission.check()?;\n let firmware', ' let firmware').replace(' admission.check()?;\n let firmware', ' let firmware').replace('admission.slice_ms()?', '10'), 'constructor wait exceeded'), + ('ignore-build-expiry', source.replace('if let Err(error) = admission.check() {', 'if let Err(error) = Ok::<(),i32>(()) {'), 'constructor must propagate'), + ] + with tempfile.TemporaryDirectory(prefix='g17-compute-constructors-') as tmp: + for name, code, expected in mutants: + if name != 'fixed': + assert code != source, f'vacuous mutation: {name}' + path = Path(tmp) / (name + '.rs') + path.write_text(code) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', '-Awarnings', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py new file mode 100644 index 0000000000000..b551cbaa96339 --- /dev/null +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run production compute admission and gate waits with a controlled clock. + +Firmware/allocation/locking boundaries are mocks. Binding theft is the review's +legal interleaving: every successful build loses its idle binding before publish. +The oracle measures one total budget, including nested waits, rather than a +fresh timeout per attempt. Mutants must fail at the named assertion. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(text, signature): + start = text.index(signature) + begin = text.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + preparation = (ROOT / 'preparation.rs').read_text() + packet = item(runtime, ' fn compute_packet(') + ensure = item(runtime, ' fn ensure_compute(') + for old, new in [('super::job::Packet', 'Packet'), + ('super::command::Validated', 'Validated'), + ('super::recovery::Memory', 'Memory'), + ('super::fw::compute_pool::required_blocks', 'required_blocks')]: + packet = packet.replace(old, new) + ensure = ensure.replace(old, new) + admission = (item(preparation, "pub(super) struct Admission<'a>") + '\n' + + item(preparation, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()') + gate = item(preparation, 'pub(super) fn enter_until(').replace('pub(super) ', '') + return r''' +#![allow(dead_code,unused_variables,irrefutable_let_patterns)] +use std::sync::{Arc,Mutex,MutexGuard,atomic::{AtomicBool,AtomicU64,AtomicUsize,Ordering}}; +type Result = std::result::Result; +const EINVAL:i32=-22;const ECANCELED:i32=-125;const EIO:i32=-5;const ENODEV:i32=-19;const EBUSY:i32=-16;const EAGAIN:i32=-11; +const RECOVERY_SETTLE_POLLS:usize=200;const WORK_RING_WAIT_NS:u64=2_000_000_000; +static NOW:AtomicU64=AtomicU64::new(0);static CALLS:AtomicUsize=AtomicUsize::new(0); +static MODE:AtomicUsize=AtomicUsize::new(0);static CANCEL:AtomicU64=AtomicU64::new(u64::MAX); +static STATUS:AtomicU64=AtomicU64::new(u64::MAX);static CLOSED:AtomicBool=AtomicBool::new(false); +fn now_ns()->u64{NOW.load(Ordering::SeqCst)} +fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);} +mod kernel{pub mod time{ + pub fn msecs_to_jiffies(ms:u32)->u32{ms} + pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms)}} + pub mod delay{pub fn fsleep(d:super::Delta){crate::advance(d.0*1_000_000);}} +}} +''' + admission + r''' +struct Wait; +impl Wait{fn lock(&self){}} +struct Changed; +impl Changed{fn wait_timeout(&self,_:&mut (),_:u32){advance(1_000_000);assert!(now_ns()<=WORK_RING_WAIT_NS,"gate wake storm must remain bounded");}} +struct Gate{wait:Wait,changed:Changed} +struct Lease; +impl Lease{fn is_current(&self)->bool{true}} +impl Gate{ + fn try_enter(self:&Arc)->Result{if CLOSED.load(Ordering::SeqCst){Err(EAGAIN)}else{Ok(Lease)}} +''' + gate + r''' +} +struct Scratch;impl Scratch{fn pages(&self)->u16{0}} +enum Validated{Compute{scratch:Scratch}} +struct Status;impl Status{fn get(&self)->i32{if now_ns()>=STATUS.load(Ordering::SeqCst){-5}else{0}}} +struct Work;impl Work{fn acquire(&self)->Result{Ok(())}} +struct Completion;impl Completion{fn status(&self)->Status{Status}fn work_state(&self)->Result{Ok(Work)}} +struct Packet{command:Validated,completion:Completion} +impl Packet{fn check_dependencies(&self)->Result{Ok(())}fn cancel_requested(&self)->bool{now_ns()>=CANCEL.load(Ordering::SeqCst)}fn is_published(&self)->bool{false}} +struct DeferredBatch;impl DeferredBatch{fn publication()->Result{Ok(Self)}fn finish(&mut self){}} +struct Recovery;impl Recovery{fn pending(&self)->bool{MODE.load(Ordering::SeqCst)==1}} +struct Memory;impl Memory{fn recovery_state(_:&())->Result{Ok(0)}} +struct Queues; +impl Queues{ + fn compute(&self,_:u64)->Option<()>{if MODE.load(Ordering::SeqCst)==0{None}else{Some(())}} + fn render_prefix(&self,_:u64,_:&Arc)->Result>{Ok(vec![])} + fn release_compute_witness(&self,_:&(),_:u64,_:&mut DeferredBatch)->Result{Ok(MODE.load(Ordering::SeqCst)==2)} +} +struct Firmware{recovery:Recovery,init:(),queues:Queues} +impl Firmware{fn replay_compute(&mut self,_:&mut DeferredBatch)->Result{Ok(())}fn publish_compute(&mut self,_:u64,_:Arc,_:&[()],_:&mut DeferredBatch)->Result{Err(EAGAIN)}} +struct State(Mutex>>);impl State{fn lock(&self)->MutexGuard<'_,Option>>{self.0.lock().unwrap()}} +struct Shared{preparations:Arc,state:State,crashed:AtomicBool,active_cores:u32}impl Shared{fn queue_events(&self){}} +struct Backend{shared:Shared,owner:u64} +fn required_blocks(_:u32,_:u16)->Result{Ok(1)} +impl Backend{ + fn admission_open(&self)->Result{Ok(())} + // The real ensure_compute forwards the original context into this allocation + // boundary. Each success loses its binding before the publication mutex. + fn build_compute(&self,_:usize,admission:&Admission<'_>)->Result{ + admission.check()?;CALLS.fetch_add(1,Ordering::SeqCst);advance(500_000_000);Ok(()) + } +''' + ensure + '\n' + packet + r''' +} +fn main(){ + let b=Backend{owner:1,shared:Shared{active_cores:1,preparations:Arc::new(Gate{wait:Wait,changed:Changed}),state:State(Mutex::new(Some(Box::new(Firmware{recovery:Recovery,init:(),queues:Queues})))),crashed:AtomicBool::new(false)}}; + let p=Arc::new(Packet{command:Validated::Compute{scratch:Scratch},completion:Completion}); + for mode in 0..4 { + MODE.store(mode,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); + let r=b.compute_packet(&p,&mut DeferredBatch); + assert_eq!(r,Err(EBUSY),"one admission budget covers binding theft/recovery/publication"); + assert!(now_ns()<=WORK_RING_WAIT_NS+10_000_000,"admission budget exceeded"); + if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),4);assert_eq!(now_ns(),WORK_RING_WAIT_NS);} + println!("PASS mode={} result={:?} calls={} elapsed_ms={}",mode,r,CALLS.load(Ordering::SeqCst),now_ns()/1_000_000); + } + for status in [false,true] { + MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); + if status{STATUS.store(10_000_000,Ordering::SeqCst);}else{CANCEL.store(10_000_000,Ordering::SeqCst);} + let r=b.compute_packet(&p,&mut DeferredBatch); + assert_eq!(r,Err(if status{EIO}else{ECANCELED}),"nested unavailable-capacity wait must check packet"); + assert_eq!(now_ns(),10_000_000,"nested unavailable-capacity cancellation/status must be prompt");assert_eq!(CALLS.load(Ordering::SeqCst),0); + STATUS.store(u64::MAX,Ordering::SeqCst);CANCEL.store(u64::MAX,Ordering::SeqCst); + } + let check=||Ok(());NOW.store(WORK_RING_WAIT_NS,Ordering::SeqCst); + let admission=Admission::new(WORK_RING_WAIT_NS,&check); + assert!(matches!(b.shared.preparations.enter_until(&admission),Err(EBUSY)),"spent budget before available gate"); + assert_eq!(b.ensure_compute(0,&admission),Err(EBUSY),"spent budget before nested admission"); + for cancel in [false,true] { + NOW.store(0,Ordering::SeqCst);CLOSED.store(true,Ordering::SeqCst); + let check=||{if cancel&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; + let admission=Admission::new(20_000_000,&check); + assert!(matches!(b.shared.preparations.enter_until(&admission),Err(e) if e==if cancel{ECANCELED}else{EBUSY}),"gate wake storm must check original deadline and cancellation"); + assert_eq!(now_ns(),if cancel{10_000_000}else{20_000_000}); + } + println!("PASS spent budgets, nested capacity cancellation/status, gate recovery wake storms"); +} +''' + + +def main(): + source = fixture() + mutants = [ + ('fixed', source, ''), + ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n loop {', 'loop {\n let admission = Admission::new(now_ns().saturating_add(WORK_RING_WAIT_NS), &check);'), 'one admission budget'), + ('omit-nested-check', source.replace(' admission.check()?;\n let pending', ' let pending'), 'nested unavailable-capacity'), + ('omit-gate-check', source.replace(' admission.check()?;\n match self.try_enter()', ' match self.try_enter()').replace('admission.slice_ms()?', '10'), 'gate wake storm'), + ] + # Bind the old unbounded mutant with the review's eventual cancellation. + mutants[1] = (mutants[1][0], mutants[1][1].replace('for mode in 0..4 {', 'CANCEL.store(10_000_000_000,Ordering::SeqCst);\n for mode in 0..4 {'), mutants[1][2]) + with tempfile.TemporaryDirectory(prefix='g17-compute-deadline-') as tmp: + for name, code, expected in mutants: + if name != 'fixed': + assert code != source, f'vacuous mutation: {name}' + path = Path(tmp) / (name + '.rs') + path.write_text(code) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', '-Awarnings', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-compute-growth-deadline-tests.py b/tools/asahi/g17-compute-growth-deadline-tests.py new file mode 100644 index 0000000000000..1e1e715de8377 --- /dev/null +++ b/tools/asahi/g17-compute-growth-deadline-tests.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Execute the complete production growth wait, abort and QID handback. + +Firmware witnesses and allocations are mocks; cancellation/expiry must abort the +transaction and drop tentative aliases before a fresh identity can be returned. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(text, signature): + start = text.index(signature) + begin = text.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +def fixture(): + runtime = (ROOT / 'runtime.rs').read_text() + prep = (ROOT / 'preparation.rs').read_text() + admission = (item(prep, "pub(super) struct Admission<'a>") + '\n' + + item(prep, "impl<'a> Admission<'a>")).replace('pub(super) ', '').replace('super::now_ns()', 'now_ns()') + methods = '\n'.join(item(runtime, sig) for sig in [' fn grow_compute(', ' fn finish_uninstalled_compute(']) + methods = methods.replace('super::object::Allocator', 'Allocator').replace('super::freelist::Pages', 'Pages') + return r''' +#![allow(dead_code)] +use std::sync::{Mutex,MutexGuard,atomic::{AtomicBool,AtomicU64,Ordering,fence}}; +type Result = std::result::Result; +type KBox = Box; +const EIO:i32=-5;const EAGAIN:i32=-11;const ENODEV:i32=-19;const EBUSY:i32=-16;const ECANCELED:i32=-125; +static NOW:AtomicU64=AtomicU64::new(0);static ALIAS:AtomicBool=AtomicBool::new(false);static PAGES:AtomicBool=AtomicBool::new(false); +static RESERVED:AtomicBool=AtomicBool::new(false);static ABORTED:AtomicBool=AtomicBool::new(false); +fn now_ns()->u64{NOW.load(Ordering::SeqCst)} +mod kernel{pub mod time{ + pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms)}} + pub mod delay{pub fn fsleep(d:super::Delta){crate::NOW.fetch_add(d.0*1_000_000,crate::Ordering::SeqCst);assert!(crate::now_ns()<100_000_000,"growth admission exceeded original deadline");}} +}} +mod g17{pub const MSG_CONTROL_NOTIFY:u32=0;} +''' + admission + r''' +struct Gate;struct Lease; +impl Gate{fn enter_until(&self,a:&Admission<'_>)->Result{a.check()?;Ok(Lease)}} +impl Lease{fn is_current(&self)->bool{true}} +struct State(Mutex>>); +impl State{fn lock(&self)->MutexGuard<'_,Option>>{self.0.lock().unwrap()}} +struct Shared{preparations:Gate,state:State,crashed:AtomicBool} +impl Shared{fn drm(&self)->Result<()>{Ok(())}fn queue_events(&self){}} +struct Context;impl Context{fn vm(&self)->&(){&()}} +struct Backend{shared:Shared,context:Context,owner:u64} +struct Allocator<'a>{dev:&'a (),uat:&'a ()} +struct Pages;struct Alias; +impl Pages{ + fn compute_sized(_:&Allocator<'_>,_:usize)->Result{PAGES.store(true,Ordering::SeqCst);Ok(Self)} + fn map_client(&self,_:&())->Result{ALIAS.store(true,Ordering::SeqCst);Ok(Alias)} +} +impl Drop for Pages{fn drop(&mut self){PAGES.store(false,Ordering::SeqCst);}} +impl Drop for Alias{fn drop(&mut self){ALIAS.store(false,Ordering::SeqCst);}} +mod compute{pub(crate) use crate::Queue;} +struct Queue{fresh:bool} +impl Queue{ + fn pool_fits(&self,_:usize)->bool{false} + fn begin_growth(&mut self,_:&(),_:usize)->Result{Ok(false)} + fn growth_token(&self)->Option{Some(7)} + fn finish_growth(&mut self,_:&(),_:&mut Option,_:&mut Option,_:usize)->Result{Err(EAGAIN)} + fn abort_growth(&mut self,t:u64)->Result>{assert_eq!(t,7);ABORTED.store(true,Ordering::SeqCst);Ok(Some(self.fresh))} +} +impl Drop for Queue{fn drop(&mut self){if self.fresh{assert!(RESERVED.load(Ordering::SeqCst));assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst),"growth aliases must precede fresh QID return");}}} +#[derive(Clone,Copy)]struct ComputeReservation; +struct Entry{reservation:ComputeReservation,queue:Option>} +struct Registry{compute:Vec>} +impl Registry{ + fn compute(&self,_:u64)->Option<&Queue>{self.compute[0].as_ref().unwrap().queue.as_deref()} + fn compute_mut(&mut self,_:u64)->Option<&mut Queue>{self.compute[0].as_mut().unwrap().queue.as_deref_mut()} + fn cancel_compute(&mut self,_:ComputeReservation)->Result{assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst));RESERVED.store(false,Ordering::SeqCst);Ok(())} +} +struct Primary;impl Primary{fn notify(&self,_:u32)->Result{Ok(())}} +struct Firmware{queues:Registry,uat:(),init:(),primary:Primary} +impl Firmware{fn scan_compute(&mut self,_:Option<()>,_:&mut DeferredBatch)->Result{Ok(())}} +struct DeferredBatch;impl DeferredBatch{fn publication()->Result{Ok(Self)}fn finish(&mut self){}} +impl Backend{ +''' + methods + r''' +} +fn main(){ + for fresh in [false,true]{for cancel in [false,true]{ + NOW.store(0,Ordering::SeqCst);ABORTED.store(false,Ordering::SeqCst);RESERVED.store(true,Ordering::SeqCst); + let b=Backend{owner:1,context:Context,shared:Shared{preparations:Gate,crashed:AtomicBool::new(false),state:State(Mutex::new(Some(Box::new(Firmware{uat:(),init:(),primary:Primary,queues:Registry{compute:vec![Some(Entry{reservation:ComputeReservation,queue:Some(Box::new(Queue{fresh}))})]}}))))}}; + let check=||{if cancel&&now_ns()>=10_000_000{Err(ECANCELED)}else{Ok(())}}; + let admission=Admission::new(20_000_000,&check); + assert_eq!(b.grow_compute(42,&admission),Err(if cancel{ECANCELED}else{EBUSY}),"growth must propagate cancel/deadline"); + assert_eq!(now_ns(),if cancel{10_000_000}else{20_000_000}); + assert!(ABORTED.load(Ordering::SeqCst));assert!(!ALIAS.load(Ordering::SeqCst)&&!PAGES.load(Ordering::SeqCst)); + assert_eq!(RESERVED.load(Ordering::SeqCst),!fresh,"only unpublished QIDs may return without firmware release"); + }} + println!("PASS 4 complete production growth waits: deadline, cancellation, installed retention, fresh rollback and tentative-alias/QID order"); +} +''' + + +def main(): + source = fixture() + mutants = [('fixed', source, ''), + ('omit-growth-check', source.replace(' admission.check()?;\n let firmware', ' let firmware'), 'growth admission exceeded'), + ('keep-growth-alias', source.replace(' drop(alias.take());', ''), 'growth aliases must precede')] + with tempfile.TemporaryDirectory(prefix='g17-compute-growth-') as tmp: + for name, code, expected in mutants: + if name != 'fixed': + assert code != source, f'vacuous mutation: {name}' + path = Path(tmp) / (name + '.rs') + path.write_text(code) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', '-Awarnings', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-growth-gate-tests.py b/tools/asahi/g17-growth-gate-tests.py index ad72c0b1c2bc6..d6ad5f9beef30 100644 --- a/tools/asahi/g17-growth-gate-tests.py +++ b/tools/asahi/g17-growth-gate-tests.py @@ -42,10 +42,10 @@ def fixture(): struct Backend {gate:Arc,ready:mpsc::Sender<()>,proceed:StdMutex>} impl Backend { fn handoff(&self,required:usize)->Result{ - let lease=self.gate.try_enter()?;let state=(); + let lease=self.gate.try_enter()?;let state=();let admission=&(); ''' + handoff + r''' } - fn grow_compute(&self,_:usize)->Result { + fn grow_compute(&self,_:usize,_:&())->Result { self.ready.send(()).unwrap();self.proceed.lock().unwrap().recv().unwrap(); let _growth=self.gate.enter(false)?; Ok(()) diff --git a/tools/asahi/g17-pool-release-tests.py b/tools/asahi/g17-pool-release-tests.py index 31c1788d74190..b7c715727acab 100644 --- a/tools/asahi/g17-pool-release-tests.py +++ b/tools/asahi/g17-pool-release-tests.py @@ -42,10 +42,15 @@ def fixture(): const ENOSPC: Error = -28; const EAGAIN: Error = -11; const FIRST_COMPUTE_QID: u8 = 4; +mod g17 { pub mod capacity { + pub enum Site { ComputeSlots } + pub fn refused(_: Site) -> i32 { -28 } +}} #[derive(Clone, Copy, PartialEq, Eq)] enum DataMaster { Compute } #[derive(Clone, Copy)] enum PoolPhase { Populated, ReleaseSent(u32), Vacant } +struct Active { len: usize } mod compute { use super::*; pub(super) struct Queue { @@ -53,12 +58,15 @@ def fixture(): pub pool_phase: PoolPhase, pub retire_pending: bool, pub released: bool, + pub quarantined: bool, + pub active: Active, } impl Queue { pub(super) fn owner(&self) -> Option { self.owner } pub(super) fn retirement_polling(&self) -> bool { self.retire_pending } PENDING } +RELEASE_PENDING } #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Id { qid: u8 } @@ -97,7 +105,7 @@ def fixture(): for slot in 0..4 { entries.push(Some(ComputeEntry { reservation: ComputeReservation { slot, id: Id { qid: slot as u8 }, owner: slot as u64 + 1 }, - queue: Some(Box::new(compute::Queue { owner: None, pool_phase: phase, retire_pending: false, released: true })), + queue: Some(Box::new(compute::Queue { owner: None, pool_phase: phase, retire_pending: false, released: true, quarantined: false, active: Active { len: 0 } })), })); } entries.push(None); @@ -118,11 +126,11 @@ def fixture(): entry.queue.as_mut().unwrap().pool_phase = PoolPhase::Vacant; } assert!(!registry.poll(), "settled idle queues must stop polling"); - assert_eq!(registry.compute_admission_error(ENOSPC), ENOSPC); + assert_eq!(registry.compute_admission_error(ENOSPC), EAGAIN); for phase in [PoolPhase::Populated, PoolPhase::Vacant] { let mut registry = state(phase, false); assert!(!registry.poll()); - assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "real exhaustion remains terminal"); + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "shared channel capacity waits for retirement"); } // Pending releases do not throttle spare physical capacity or hide errors. let mut registry = state(PoolPhase::ReleaseSent(u32::MAX), true); @@ -140,12 +148,22 @@ def fixture(): assert!(registry.poll(), "existing command-retirement polling remains enabled"); // An exited owner's command waiting for its stamp settles without a new event. assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "a settling released queue is temporary backpressure"); - // A command still owned by a live client is not a release. + // A closed owner retains its identity until its final command retires. registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().owner = Some(1); - assert_eq!(registry.reserve_compute(200), Err(ENOSPC), "an owned retiring queue does not hide exhaustion"); + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().active.len = 1; + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "closed owner still has a command"); + // An off-lock graph constructor also returns capacity or installs a queue. + let mut building = state(PoolPhase::Populated, false); + building.compute[0].as_mut().unwrap().queue = None; + assert_eq!(building.reserve_compute(200), Err(EAGAIN), "constructor can return capacity"); + building.compute.pop(); + assert_eq!(building.reserve_compute(200), Err(EAGAIN), "full registry must also wait for a constructor"); + // A command still owned by a live client is not a release. + registry.compute[0].as_mut().unwrap().queue.as_mut().unwrap().released = false; + assert_eq!(registry.reserve_compute(200), Err(EAGAIN), "an owned retiring queue can yield its channel after its witness"); println!("PASS captured four-queue transition, settled wake removal, stamp-wait settling, full/free identity capacity and error preservation"); } -'''.replace('PENDING', pending).replace('METHODS', methods).replace('PREDICATE', predicate) +'''.replace('RELEASE_PENDING', item(compute, 'fn release_pending(')).replace('PENDING', pending).replace('METHODS', methods).replace('PREDICATE', predicate) def main(): @@ -158,6 +176,8 @@ def main(): current = current.replace('queue.retirement_polling() || queue.pool_release_pending()', 'queue.retirement_polling()') elif mutation == 'terminal-admission': current = current.replace('.map_err(|error| self.compute_admission_error(error))?', '?') + if mutation != 'fixed': + assert current != code, f'vacuous mutation: {mutation}' path = root / (mutation + '.rs') path.write_text(current) binary = path.with_suffix('') @@ -167,7 +187,10 @@ def main(): assert result.returncode == 0, result.stderr print(result.stdout.strip()) else: - expected = 'release needs a worker wake' if mutation == 'no-wake' else 'pending release is temporary' + expected = { + 'no-wake': 'release needs a worker wake', + 'terminal-admission': 'pending release is temporary', + }[mutation] assert result.returncode != 0 and expected in result.stderr, result.stderr print('PASS negative control:', mutation) diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 73d5fa52b9833..36d4ced3510bb 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -24,6 +24,11 @@ def item(start): limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) header='''#![allow(dead_code, unused_macros)] type Result = std::result::Result; + mod g17 { pub mod capacity { + pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } + pub fn refused(_: Site) -> i32 { -28 } + }} + const ENOSPC:i32=-28; const EIO:i32=-5; macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } type KVec = Vec; struct InFlight; @@ -31,6 +36,20 @@ def item(start): methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' fn main(){ + if cfg::RENDER_PAIRS_PER_OWNER == 1 { + let mut state=State::new(); + let mut slots=Vec::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=state.acquire(owner).expect("every logical owner must fit");slots.push(slot); + assert_eq!(state.owners.iter().filter(|id|**id==owner).count(),1); + } + for slot in slots {state.release(slot);} + let mut state=State::new();let first=state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1),Some(first));} + assert_eq!(state.acquire(1),None);state.release(first);assert_eq!(state.acquire(1),Some(first)); + println!("shared render admission: all58 owners fit; depth16 waits for exact retirement without consuming a spare"); + return; + } let mut state=State::new(); for owner in 1..=32 { assert!(state.refusal(owner).is_ok()); @@ -118,6 +137,12 @@ def item(s,start): constants=re.search(r'const QID_MAX: [^;]+;',fw).group(0)+'\n'+'\n'.join(re.search(r'const '+name+r': [^;]+;',s).group(0) for name in ['QID_COUNT','COMPUTE_RENDER_RESERVE']) header='''#![allow(dead_code, unused_macros)] type Result = std::result::Result; + mod g17 { pub mod capacity { + pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } + pub fn refused(_: Site) -> i32 { -28 } + }} + + mod mmu { pub const MAX_EXECUTION_CONTEXTS:u32=58; } const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} @@ -167,8 +192,8 @@ def item(s,start): assert_eq!(q.entries[0].unwrap().id,protected); // Replacement must not consume the protected render minimum. let mut q=fresh();q.reserve_initial_render().unwrap(); - for owner in 3..17 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} - for owner in 1000..1112 {q.reserve(owner,DataMaster::Compute,None).unwrap();} + for owner in 3..=COMPUTE_RENDER_RESERVE as u64 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 1000..1000+(QID_COUNT-COMPUTE_RENDER_RESERVE) as u64 {q.reserve(owner,DataMaster::Compute,None).unwrap();} assert_eq!(q.leased,QID_COUNT);assert_eq!(q.reserve(2000,DataMaster::Compute,None),Err(ENOSPC)); assert!(q.initial_render.iter().all(Option::is_some)); // Ordinary free capacity leaves the initialization reservations untouched. @@ -218,6 +243,7 @@ def shared_alias_tests(root, source): use std::{cell::RefCell,rc::Rc,ops::Range}; type Result = std::result::Result;type Error=i32; const EINVAL:i32=-22;const EOVERFLOW:i32=-75;const ENOSPC:i32=-28;const EIO:i32=-5; +mod g17 {pub mod capacity {pub enum Site {LowerVa, AliasVa} pub fn refused(_: Site)->i32 {-28}}} mod hw {pub mod t8140 {pub mod dynamic {pub const LOWER:std::ops::Range=0..256;}}} mod mmu {pub const UAT_PGSZ:usize=16;pub type Prot=u8;pub type Vm=super::Vm;pub type KernelMapping=super::Mapping;} #[derive(Clone,Copy)] enum CpuMap {WriteCombined} diff --git a/tools/asahi/g17-shared-pool-tests.py b/tools/asahi/g17-shared-pool-tests.py new file mode 100644 index 0000000000000..9c49872c01c38 --- /dev/null +++ b/tools/asahi/g17-shared-pool-tests.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile production pressure reclaim with mock firmware/mapping boundaries. + +The production eligibility loop must retain backing until release consumption, +exclude every borrower and defer mapping destruction until outside the lock. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + begin = source.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def fixture(): + method = item((ROOT / 'runtime/render.rs').read_text(), 'pub(super) fn reclaim_idle_render_pools(').replace('pub(super) ', '') + return r''' +#![allow(dead_code)] +use std::sync::{Arc, Mutex, atomic::{AtomicUsize, Ordering}}; +type Result = std::result::Result; +const EIO:i32=-5; +mod g17 {pub mod initdata {pub struct InitData;}} +struct Backing(Arc); +impl Backing {fn new(used:&Arc)->Self {used.fetch_add(1,Ordering::SeqCst);Self(used.clone())}} +impl Drop for Backing {fn drop(&mut self) {self.0.fetch_sub(1,Ordering::SeqCst);}} +struct Pool {id:u16, backing:Mutex>} +impl Pool { + fn id(&self)->u16 {self.id} + fn control_va(&self)->u64 {0} + fn vacant(&self)->bool {self.backing.lock().unwrap().is_none()} + fn vacate(&self)->Option {self.backing.lock().unwrap().take()} +} +struct Context; +struct Pair {idle:Result} +impl Pair {fn context(&self)->&Context {&Context} fn idle_for_context(&self,_:&Context)->Result{self.idle}} +struct Reservation {owner:u64} +struct Entry {reservation:Reservation,pool:Option>,pair:Option,constructing:bool,growing:bool,preparing:Option<()>,pending_error:Option<()>,pending_timeout:bool} +struct Pools {consumed:bool,registered:bool} +impl Pools {fn observe(&mut self,_:&g17::initdata::InitData)->Result{Ok(())} fn released(&self,_:u32,_:u64)->bool{!self.registered||self.consumed}} +struct Render {entries:Vec>,pools:Pools} +struct Admission {active:bool} +impl Admission {fn owner_active(&self,_:u64)->bool {self.active}} +struct Registry {render:Render,admission:Admission} +struct DeferredBatch {backing:Vec,cap:usize} +impl DeferredBatch { + fn render_backing_room(&self)->bool{self.backing.len(),count:usize)->Registry { + let entries=(0..count).map(|id|Some(Entry { + reservation:Reservation{owner:id as u64+1}, + pool:Some(Arc::new(Pool{id:id as u16,backing:Mutex::new(Some(Backing::new(used)))})), + pair:Some(Pair{idle:Ok(true)}),constructing:false,growing:false,preparing:None,pending_error:None,pending_timeout:false, + })).collect(); + Registry {render:Render{entries,pools:Pools{consumed:true,registered:true}},admission:Admission{active:false}} +} +fn main(){ + let used=Arc::new(AtomicUsize::new(0)); + for guard in 0..10 { + let mut registry=state(&used,1); + let mut deferred=DeferredBatch{backing:vec![],cap:4}; + match guard { + 0=>registry.render.pools.consumed=false, + 1=>registry.admission.active=true, + 2=>registry.render.entries[0].as_mut().unwrap().constructing=true, + 3=>registry.render.entries[0].as_mut().unwrap().growing=true, + 4=>registry.render.entries[0].as_mut().unwrap().preparing=Some(()), + 5=>registry.render.entries[0].as_mut().unwrap().pending_error=Some(()), + 6=>registry.render.entries[0].as_mut().unwrap().pending_timeout=true, + 7=>registry.render.entries[0].as_mut().unwrap().pair.as_mut().unwrap().idle=Ok(false), + 8=>deferred.cap=0, + 9=>registry.render.entries.push(Some(Entry{reservation:Reservation{owner:1},pool:None,pair:None,constructing:true,growing:false,preparing:None,pending_error:None,pending_timeout:false})), + _=>unreachable!(), + } + registry.reclaim_idle_render_pools(&g17::initdata::InitData,&mut deferred).unwrap(); + assert!(deferred.backing.is_empty(),"guard {} must retain pool backing",guard); + assert!(!registry.render.entries[0].as_ref().unwrap().pool.as_ref().unwrap().vacant()); + assert_eq!(used.load(Ordering::SeqCst),1); + drop(registry);assert_eq!(used.load(Ordering::SeqCst),0); + } + for _ in 0..512 { + let mut registry=state(&used,4); + let mut deferred=DeferredBatch{backing:vec![],cap:4}; + registry.reclaim_idle_render_pools(&g17::initdata::InitData,&mut deferred).unwrap(); + assert_eq!(deferred.backing.len(),4); + assert_eq!(used.load(Ordering::SeqCst),4,"mapping destruction must remain off-lock"); + registry.reclaim_idle_render_pools(&g17::initdata::InitData,&mut deferred).unwrap(); + assert_eq!(deferred.backing.len(),4,"vacant pools cannot double-return"); + deferred.finish();assert_eq!(used.load(Ordering::SeqCst),0); + } + let mut unregistered=state(&used,1); + unregistered.render.pools.registered=false;unregistered.render.pools.consumed=false; + let mut deferred=DeferredBatch{backing:vec![],cap:4}; + unregistered.reclaim_idle_render_pools(&g17::initdata::InitData,&mut deferred).unwrap(); + assert_eq!(deferred.backing.len(),1,"never-registered backing has no firmware reference"); + deferred.finish();assert_eq!(used.load(Ordering::SeqCst),0); + println!("PASS 512 four-client pool-sharing cycles and ten witness/borrower/collector guards; backing returns to baseline off-lock"); +} +''' + + +def main(): + source = fixture() + mutations = [('fixed', source), + ('ignore-release-consumption', source.replace('!render.pools.released(u32::from(pool.id()), pool.control_va())', 'false')), + ('ignore-admission-borrower', source.replace('self.admission.owner_active(entry.reservation.owner)', 'false')), + ('ignore-constructor', source.replace('other.constructing', 'false'))] + with tempfile.TemporaryDirectory(prefix='g17-shared-pools-') as tmp: + for name, code in mutations: + path = Path(tmp) / (name + '.rs') + binary = path.with_suffix('') + path.write_text(code) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and 'must retain pool backing' in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-unpublished-tests.py b/tools/asahi/g17-unpublished-tests.py new file mode 100644 index 0000000000000..a162b2651a037 --- /dev/null +++ b/tools/asahi/g17-unpublished-tests.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Inject a nonfatal first-submit error while all ten logical owners stay open.""" +from pathlib import Path +import importlib.util +import os +import subprocess +import tempfile +ROOT=Path(__file__).resolve().parents[2]/'drivers/gpu/drm/asahi/g17' +spec=importlib.util.spec_from_file_location('fault',Path(__file__).with_name('g17-capacity-fault-tests.py')) +adapter=importlib.util.module_from_spec(spec);spec.loader.exec_module(adapter) + + +def fixture(): + text=adapter.fixture().split('fn main() {')[0] + method=adapter.item((ROOT/'queue/compute.rs').read_text(),'pub(crate) fn rollback_unpublished(') + finish=adapter.item((ROOT/'runtime/backend.rs').read_text(),'pub(super) fn finish_publication(').replace('pub(super) ','') + return text+'\nimpl Queue {\n'+method+'\n}\n'+r''' +const ETIMEDOUT:i32=-110;const ENODEV:i32=-19;const ENOMEM:i32=-12; +type Fence=u8; +#[derive(PartialEq)] enum Engine{Compute} +struct Status; +impl Status {fn get(&self)->i32{0}} +impl Completion {fn status(&self)->Status{Status} fn fail_unpublished(&self,_:i32){} fn scheduler_fence(&self)->Fence{0}} +impl Packet {fn is_published(&self)->bool{false} fn engine(&self)->Engine{Engine::Compute}} +struct DeferredBatch; +impl DeferredBatch {fn finish(&mut self){}} +mod job {pub trait Backend{fn fail_vm(&self,error:i32);}} +struct Backend {q:std::cell::RefCell,owner:u64,context:Arc} +impl job::Backend for Backend {fn fail_vm(&self,_:i32){panic!("nonfatal first submit must leave logical owner open")}} +impl Backend { + fn stop_after_error(&self){panic!("rollback must preserve healthy device")} + fn rollback_unpublished_compute(&self)->Result { + let (cancel,binding)={self.q.borrow_mut().rollback_unpublished(self.owner,&self.context)?}; + assert!(!cancel);drop(binding);Ok(()) + } +''' + finish + r''' +} +fn main(){ + let live=Arc::new(AtomicUsize::new(0)); + for predecessor in 0..3 { + let mut channels=Vec::new(); + for owner in 1..=10 { + let mut q=idle(&live);q.owner=Some(owner);q.unpublished_binding=true; + let context=q.binding.as_ref().unwrap().context.clone(); + if predecessor!=0 { + let old=Context::new(100+owner,&live); + q.previous=Some(Previous{binding:if predecessor==1{Some(Binding{context:old.clone()})}else{None},context:old,publication:None}); + } + channels.push(Backend{q:std::cell::RefCell::new(q),owner,context}); + } + let packet=Arc::new(Packet{completion:Arc::new(Completion)}); + for b in &channels { + assert_eq!(b.finish_publication(&packet,Err(ENOMEM),&mut DeferredBatch),Err(ENOMEM)); + let q=b.q.borrow();assert!(q.reusable(),"failed open-owner first submit must return channel"); + assert!(q.owner.is_none());assert!(q.previous.is_none());assert_eq!(q.graph.resets,0); + } + drop(channels);assert_eq!(live.load(Ordering::SeqCst),0); + } + for guard in 0..13 { + let mut q=idle(&live);q.unpublished_binding=true; + let context=q.binding.as_ref().unwrap().context.clone();let mut owner=2;let mut ctx=context.clone(); + match guard { + 0=>q.unpublished_binding=false, + 1=>q.exit=Some(Exit{phase:Phase::Acknowledged}), + 2=>q.active.push(Active{packet:Arc::new(Packet{completion:Arc::new(Completion)}),ticket:None,qos_pending:false}).unwrap(), + 3=>q.replays.push(Arc::new(Packet{completion:Arc::new(Completion)})).unwrap(), + 4=>q.growing=true, + 5=>q.quarantined=true, + 6=>q.awaiting_witness=true, + 7=>q.previous=Some(Previous{context:context.clone(),binding:None,publication:Some((1,1))}), + 8=>q.pool_phase=PoolPhase::ReleaseSent(3), + 9=>owner=3, + 10=>ctx=Context::new(4,&live), + 11=>q.retire_pending=true, + 12=>q.failure_status_pending=true, + _=>unreachable!(), + } + let (cancel,binding)=q.rollback_unpublished(owner,&ctx).unwrap(); + assert!(!cancel);assert!(binding.is_none());assert_eq!(q.owner,Some(2),"guard {} must retain physical owner",guard); + } + let mut q=idle(&live);q.installed=false;q.unpublished_binding=true; + let context=q.binding.as_ref().unwrap().context.clone(); + assert!(q.rollback_unpublished(2,&context).unwrap().0,"never-published graph can cancel only its unpublished QID"); + println!("PASS production failure completion returns all ten open-owner channels for detached/same-VM/rebound predecessors; 13 published/kill/work/owner/epoch guards retain ownership; installed rings never reset"); +} +''' + + +def main(): + source=fixture() + compute=(ROOT/'queue/compute.rs').read_text() + assert compute.count('visible = true;\n self.unpublished_binding = false;')==2 + backend=(ROOT/'runtime/backend.rs').read_text() + assert 'if !lease.is_current() { drop(state); drop(lease); continue; }' in backend + cases=[('fixed',source),('omit-first-failure-rollback',source.replace('self.rollback_unpublished_compute().is_err()','false')), + ('rollback-published-binding',source.replace('|| !self.unpublished_binding','')), + ('rollback-issued-kill',source.replace('|| self.exit.is_some() || self.active.len != 0','|| self.active.len != 0'))] + with tempfile.TemporaryDirectory(prefix='g17-unpublished-') as tmp: + for name,code in cases: + path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021','-Awarnings',str(path),'-o',str(binary)],check=True) + run=subprocess.run([str(binary)],capture_output=True,text=True,timeout=15) + if name=='fixed': + assert run.returncode==0,run.stderr + print(run.stdout.strip()) + else: + assert run.returncode!=0,name + print('PASS negative control:',name) +if __name__=='__main__':main() From b52bd36805e0c35bac5c8255aae8cfad7cce5dd7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 11:47:00 +1100 Subject: [PATCH 08/11] drm/asahi: give native ray-query jobs a bounded execution allowance Keep the two-second scheduler poll and ordinary-job deadline. Validated intersection state identifies compute and render ray traversal; allow it four extra polls without resetting replay or cross-VM credits. The ten-second allowance remains far below the legacy scheduler fallback and cannot be extended indefinitely by the ray-query tag. Exercise the production policy on the host, including exhaustion, ordinary and unpublished jobs, and the independent cross-VM bound. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/job.rs | 15 +++++- drivers/gpu/drm/asahi/g17/timeout.rs | 58 ++++++++++++++++++++++ tools/asahi/g17-ray-query-timeout-tests.sh | 10 ++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100755 tools/asahi/g17-ray-query-timeout-tests.sh diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 083cfe3d1ce29..f49a3effed8bd 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -272,6 +272,7 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, + ray_query_renewals: u8, packet: Arc, } @@ -295,6 +296,7 @@ impl sched::JobImpl for Job { Timeout { backend: self.backend.clone(), victim_renewals: 0, + ray_query_renewals: 0, packet: self.packet.clone(), } } @@ -303,6 +305,17 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } + let ray_query = match &job.packet.command { + Validated::Compute { intersection, .. } => intersection.enabled(), + Validated::Render { ray_query, .. } => { + ray_query.iter().any(|stage| stage.intersection.enabled()) + } + }; + if super::timeout::renew_ray_query( + job.packet.is_published(), ray_query, &mut job.ray_query_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred && job.packet.engine() == Engine::Compute { job.victim_renewals += 1; diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index e76dab07f56db..ddc052e8fd947 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,6 +6,22 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. +/// Native ray traversal can legitimately exceed one poll (CTS stress). Four +/// extra polls give it ten seconds, still one tenth of the legacy scheduler's +/// fallback. This is an execution allowance, not evidence of GPU progress, and +/// is consumed once per scheduler job, independently of cross-VM/replay grace. +pub(crate) const POLL_MS: u32 = 2000; +const RAY_QUERY_RENEWALS: u8 = 4; + +pub(crate) fn renew_ray_query(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= RAY_QUERY_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { pub(crate) vm: usize, @@ -39,6 +55,48 @@ mod tests { Head { vm, published, finished } } #[test] + fn ray_query_execution_is_bounded_at_ten_seconds() { + let mut renewals = 0; + for poll in 1..=4 { + assert!(renew_ray_query(true, true, &mut renewals)); + assert_eq!(renewals, poll); + } + for _ in 0..256 { + assert!(!renew_ray_query(true, true, &mut renewals)); + } + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); + } + #[test] + fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { + for (published, enabled) in [(false, false), (false, true), (true, false)] { + let mut renewals = 0; + assert!(!renew_ray_query(published, enabled, &mut renewals)); + assert_eq!(renewals, 0); + } + } + #[test] + fn ray_query_and_cross_vm_credits_are_independent_and_finite() { + let mut ray = 0; + let own = head(5, 101, false); + for _ in 0..4 { + assert!(renew_ray_query(true, true, &mut ray)); + } + for victim in 0..MAX_RENEWALS { + assert!(!renew_ray_query(true, true, &mut ray)); + assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); + } + #[test] + fn exhausted_or_invalid_credit_never_wraps() { + for value in 4..=u8::MAX { + let mut renewals = value; + assert!(!renew_ray_query(true, true, &mut renewals)); + assert_eq!(renewals, value); + } + } + #[test] fn earlier_other_vm_gets_bounded_grace() { let own = head(5, 101, false); for count in 0..MAX_RENEWALS { diff --git a/tools/asahi/g17-ray-query-timeout-tests.sh b/tools/asahi/g17-ray-query-timeout-tests.sh new file mode 100755 index 0000000000000..9aaa3aa653c14 --- /dev/null +++ b/tools/asahi/g17-ray-query-timeout-tests.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Host-only tests of the production policy. Keep outputs in the caller-owned dir. +set -eu +out=${1:?usage: g17-ray-query-timeout-tests.sh OUTPUT_DIRECTORY} +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +mkdir -p "$out" +"${RUSTC:-rustc}" --edition=2021 --test \ + "$root/drivers/gpu/drm/asahi/g17/timeout.rs" -o "$out/g17-timeout-tests" +"$out/g17-timeout-tests" From a5bc25b2ce1997452676894fc8c9b401c7396e6c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 16:26:22 +1100 Subject: [PATCH 09/11] drm/asahi: cover render helpers and AS builds with bounded execution 73f already recognizes both render intersection stages, but untagged helper and transfer jobs ordered behind native traversal can still hit the other engine watchdog after two seconds. Select the allowance from the entire validated submission before enqueueing its first command. Keep ordinary submissions and all admission and ownership gates intact. AS build kernels do not carry intersection state. Add a feature-gated AS_BUILD bit in the existing compute flags field; it grants only the same fixed four additional polls. Reject all other unsupported flags. Port the production timeout tests to KUnit and cover selection, helper scope, both render halves, build hints and finite exhaustion. Document CTS scale, throughput thresholds and the limits of the macOS comparison. Hardware attribution and completion remain to be qualified separately. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../gpu/asahi-g17-execution-allowance.rst | 98 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/command.rs | 24 ++++- drivers/gpu/drm/asahi/g17/job.rs | 14 +-- drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/timeout.rs | 75 +++++++++++--- include/uapi/drm/asahi_drm.h | 17 ++++ 7 files changed, 211 insertions(+), 25 deletions(-) create mode 100644 Documentation/gpu/asahi-g17-execution-allowance.rst diff --git a/Documentation/gpu/asahi-g17-execution-allowance.rst b/Documentation/gpu/asahi-g17-execution-allowance.rst new file mode 100644 index 0000000000000..c08e54d74c10a --- /dev/null +++ b/Documentation/gpu/asahi-g17-execution-allowance.rst @@ -0,0 +1,98 @@ +Bounded G17 execution allowance +============================== + +Selection and scope +------------------- + +73f already selected native ray queries in Compute and both Render halves. +It did not cover untagged helper/transfer commands ordered behind them on +another engine. The scheduler watchdog is per engine: an untagged follower +can reach its two-second poll while a legitimate long producer still runs. +The failure location in a runtime settlement path is not itself proof that +the long producer hung. This change addresses that selection gap; the exact +reported hardware failure attribution still requires qualification. + +After every command and destination in one submit ioctl has been validated, +select a long execution allowance if any command contains validated native +intersection state (CDM, VDM or fragment), or the explicit AS-build hint. +Apply that selection to every command in that ioctl before enqueueing any. +VDM covers vertex execution; tessellation and geometry stages may also use +CDM pre-graphics streams, so both engines must participate. An untagged +ordered transfer/helper in that submission receives the same finite budget. +An ordinary submission retains its original deadline. Selection never changes +VM access checks, firmware dependencies, fences, admission, teardown, recovery +ownership or resource limits, and does not persist into another ioctl. + +Build kernels use ordinary compute and compute-based blits, not intersection +state. Neither SCS allocation, attachment count, stream byte size nor the +existing WIDE_VISIBILITY flag identifies an AS build reliably. Use the +existing per-command compute flags field with DRM_ASAHI_COMPUTE_AS_BUILD, +advertised only on G17 by DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD. Older kernels +reject that flag; Mesa must test the feature before setting it. This is an +untrusted hint, never proof that the stream builds an AS or makes progress. +All existing command/resource validation still runs. Reserved/unknown flags +remain rejected. Userspace can at most obtain the same fixed allowance as +native ray-query metadata already permits. + +Bound and sizing +---------------- + +Keep the scheduler interval at 2,000 ms. Only a published selected job can +consume four additional polls: (1 + 4) * 2,000 = 10,000 ms. The counter belongs +to the scheduler job and never resets at a replay, wake or resubmission within +that job. Exhausted and invalid counters cannot wrap. Ordinary/unpublished +jobs cannot consume this credit. Cross-VM attribution has its own pre-existing +8-credit bound: at most 26,000 ms of combined poll budgets if all 4 execution +and 8 cross-VM credits are consumed. These are watchdog policy budgets, not +hard real-time wall-clock guarantees (workqueue scheduling can delay a poll). +Existing replay/retirement grace is independent; this patch does not alter it. + +The public CTS ray-query stress scene has 20,000 levels * 3 primitives = +60,000 primitives and 60,000 result rays. Ten seconds requires 6,000 rays/s; +two seconds requires 30,000 rays/s. In an adversarial all-overlap traversal, +60,000 * 60,000 is 3.6 billion potential primitive visits, requiring 360 +million visits/s at ten seconds versus 1.8 billion/s at two seconds. This is +a conservative sizing model, not a measured G17P rate or an assertion that +all tests perform every possible visit. Native traversal includes generated +shader, candidate and continuation work, so hardware peak ray throughput +cannot be substituted for end-to-end CTS throughput. + +The requested million-geometry build scale (1,048,576 geometries) requires +104,858 geometries/s at ten seconds versus 524,288/s at two seconds, before +accounting for geometry uploads, leaf generation, Morton sorting and native +encoding. Geometry count and primitive count are distinct; this is not a +universal throughput promise. Retaining the established 10-second allowance +provides 5x the old budget without giving arbitrary shaders an unlimited +watchdog exemption. Splitting or optimizing larger builds remains necessary +if real jobs exceed it. No new hardware measurements were made for this patch. + +Comparison +---------- + +The G13-G15 Asahi scheduler fallback in queue/mod.rs is 100,000 ms for all +jobs, including software ray traversal and build compute. The G17 execution +allowance is one tenth of that scheduler fallback and ordinary G17 work +continues to use 2,000 ms. Firmware fault handling is separate from the host +scheduler fallback, on both paths. + +Apple documents MTLCommandBufferError.timeout for command buffers that take +too long. Its public API does not specify a portable numeric macOS watchdog +limit or an entitlement to unlimited GPU execution. Therefore 10 seconds here +is a Linux policy choice, not a claim of macOS parity or a macOS guarantee. + +Public references: + +* https://github.com/KhronosGroup/VK-GL-CTS/blob/main/external/vulkancts/modules/vulkan/ray_query/vktRayQueryStressTests.cpp +* https://developer.apple.com/documentation/metal/mtlcommandbuffererror-swift.struct/timeout + +Validation +---------- + +asahi_g17_execution_allowance KUnit and the host runner execute the same +production selection, flag admission and finite-credit functions. Cases cover +CDM traversal, each render half, AS builds without intersection state, mixed +helper submissions, ordinary following submissions, unpublished jobs, +exhaustion/overflow and independent cross-VM attribution credits. Software +gates cannot prove completion or correctness of the seven graphics stress +cases or the 29 large AS-build cases; qualify those under a separate approved +Neo hardware task using matching Mesa and kernel artifacts. diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index ae6a5d8a49859..26785a326c6f0 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,6 +12,8 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; +const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); + const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -306,12 +308,14 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { + long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -323,6 +327,19 @@ pub(crate) enum Validated { }, } +impl Validated { + pub(crate) fn long_job(&self) -> bool { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, + } + } + pub(crate) fn allow_long_job(&mut self) { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, + } + } +} + impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -447,6 +464,8 @@ impl Payload { } } Ok(Validated::Render { + long_job: super::timeout::select(false, + ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -454,8 +473,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 + let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; + if cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -476,6 +495,7 @@ impl Payload { if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { + long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index f49a3effed8bd..27986b60d47b2 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -272,7 +272,7 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, - ray_query_renewals: u8, + execution_renewals: u8, packet: Arc, } @@ -296,7 +296,7 @@ impl sched::JobImpl for Job { Timeout { backend: self.backend.clone(), victim_renewals: 0, - ray_query_renewals: 0, + execution_renewals: 0, packet: self.packet.clone(), } } @@ -305,14 +305,8 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } - let ray_query = match &job.packet.command { - Validated::Compute { intersection, .. } => intersection.enabled(), - Validated::Render { ray_query, .. } => { - ray_query.iter().any(|stage| stage.intersection.enabled()) - } - }; - if super::timeout::renew_ray_query( - job.packet.is_published(), ray_query, &mut job.ray_query_renewals, + if super::timeout::renew_execution( + job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, ) { return true; } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6e0295e721d42..9129418f4a16f 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1099,7 +1099,8 @@ impl gpu::Gpu for Gpu { | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index f9201f6391b38..fb010ff459df1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -391,7 +391,10 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes, objects)?; + let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + } let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ddc052e8fd947..df6656070a373 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -7,15 +7,36 @@ pub(crate) const MAX_RENEWALS: u8 = 8; /// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. -/// Native ray traversal can legitimately exceed one poll (CTS stress). Four +/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four /// extra polls give it ten seconds, still one tenth of the legacy scheduler's /// fallback. This is an execution allowance, not evidence of GPU progress, and /// is consumed once per scheduler job, independently of cross-VM/replay grace. pub(crate) const POLL_MS: u32 = 2000; -const RAY_QUERY_RENEWALS: u8 = 4; +const EXECUTION_RENEWALS: u8 = 4; -pub(crate) fn renew_ray_query(published: bool, enabled: bool, renewals: &mut u8) -> bool { - if !published || !enabled || *renewals >= RAY_QUERY_RENEWALS { +/// Input comes only from fully validated commands. Render metadata covers the +/// VDM half (vertex/tessellation/geometry) and the fragment half independently. +/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. +pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { + compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build +} + +/// Firmware dependencies can hold an untagged helper behind a long command on +/// the other engine. Classify the complete validated ioctl before enqueueing +/// anything; no state or allowance leaks into a later submission. +pub(crate) fn select_submission(commands: impl Iterator) -> bool { + commands.into_iter().any(|enabled| enabled) +} + +/// Only the explicit AS-build bit is newly accepted; reserved and unrelated +/// flags remain invalid on G17P. Return None before any command is published. +pub(crate) fn as_build(flags: u32) -> Option { + const AS_BUILD: u32 = 1 << 2; + if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } +} + +pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= EXECUTION_RENEWALS { return false; } *renewals += 1; @@ -48,9 +69,41 @@ pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator Head { Head { vm, published, finished } } @@ -58,11 +111,11 @@ mod tests { fn ray_query_execution_is_bounded_at_ten_seconds() { let mut renewals = 0; for poll in 1..=4 { - assert!(renew_ray_query(true, true, &mut renewals)); + assert!(renew_execution(true, true, &mut renewals)); assert_eq!(renewals, poll); } for _ in 0..256 { - assert!(!renew_ray_query(true, true, &mut renewals)); + assert!(!renew_execution(true, true, &mut renewals)); } assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); } @@ -70,7 +123,7 @@ mod tests { fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { for (published, enabled) in [(false, false), (false, true), (true, false)] { let mut renewals = 0; - assert!(!renew_ray_query(published, enabled, &mut renewals)); + assert!(!renew_execution(published, enabled, &mut renewals)); assert_eq!(renewals, 0); } } @@ -79,10 +132,10 @@ mod tests { let mut ray = 0; let own = head(5, 101, false); for _ in 0..4 { - assert!(renew_ray_query(true, true, &mut ray)); + assert!(renew_execution(true, true, &mut ray)); } for victim in 0..MAX_RENEWALS { - assert!(!renew_ray_query(true, true, &mut ray)); + assert!(!renew_execution(true, true, &mut ray)); assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); } assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); @@ -92,7 +145,7 @@ mod tests { fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; - assert!(!renew_ray_query(true, true, &mut renewals)); + assert!(!renew_execution(true, true, &mut renewals)); assert_eq!(renewals, value); } } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index d311b92620b86..a482ab38b5952 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -348,6 +348,12 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, + + /** + * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept + * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. + */ + DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -1614,6 +1620,17 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + + /** + * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, + * including compute-based transfers. Requires BOUNDED_AS_BUILD. + * This is an execution hint, not proof of progress or permission to + * access additional memory. The kernel validates the command normally + * and caps its execution allowance at ten seconds (four extra 2s polls). + * Other commands in the same submission receive the same allowance so + * ordered helpers cannot time out while waiting for the build. + */ + DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** From 9827b6bb277af5912490229f6fe56d3c575dafe3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 16:42:05 +1100 Subject: [PATCH 10/11] drm/asahi: bound repeated timeout recovery and G17 mailbox stalls Prevent compute replay from waiting indefinitely for event-serviced input fences while holding the device mutex. Bound initial dependency waits and recovery preparation joins. Queue workload-triggered stop from scheduler callbacks; halt firmware producers before a bounded stop join and retain all graph/lease ownership when a borrower has not returned. Stop on a per-device burst of eight watchdog expirations or eight restart handshakes within thirty seconds. New VMs, generations and replay cannot renew that budget or undo terminal loss. Charge render borrowed-graph and queued-retirement deferrals against the same finite eight-poll bound as compute attribution; a deferred timeout cannot evade eventual device loss. G17 RX now uses an oneshot threaded interrupt with a 64-message pass and an off-lock sleep on exhaustion. G17 non-atomic TX without an IRQ sleeps instead of polling for 500 ms with interrupts disabled; atomic sends have a 1 ms cap. Share one absolute TX deadline across contention/retries and preserve provider/removal/PM/cookie gates. Add lockdep KUnit for 64 consecutive timeouts during pending recovery, late borrower return and latched failure; inject permanently full/empty register models through production mailbox poll/send functions. Document the audited bounds and the unverified hardware attribution. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../gpu/asahi-g17-recovery-safety.rst | 72 ++++++++++++++++ drivers/gpu/drm/asahi/g17/dependency.rs | 8 ++ drivers/gpu/drm/asahi/g17/job.rs | 45 ++++++---- drivers/gpu/drm/asahi/g17/preparation.rs | 11 +++ drivers/gpu/drm/asahi/g17/recovery.rs | 7 ++ drivers/gpu/drm/asahi/g17/recovery/safety.rs | 84 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 32 +++++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 7 +- .../drm/asahi/g17/runtime/render/observe.rs | 2 +- drivers/gpu/drm/asahi/g17/timeout.rs | 10 +++ drivers/soc/apple/mailbox-g17-test.h | 66 +++++++++++++++ drivers/soc/apple/mailbox.c | 71 +++++++++++++--- 14 files changed, 392 insertions(+), 30 deletions(-) create mode 100644 Documentation/gpu/asahi-g17-recovery-safety.rst create mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs create mode 100644 drivers/soc/apple/mailbox-g17-test.h diff --git a/Documentation/gpu/asahi-g17-recovery-safety.rst b/Documentation/gpu/asahi-g17-recovery-safety.rst new file mode 100644 index 0000000000000..f56d08ac2e570 --- /dev/null +++ b/Documentation/gpu/asahi-g17-recovery-safety.rst @@ -0,0 +1,72 @@ +G17 repeated-timeout containment +=============================== + +The 73f audit found workload-reachable host-liveness hazards. The journal +alone cannot attribute the observed machine reset to one of them; no hardware +reproduction is claimed here. + +* Compute replay called Packet::check_dependencies with the device mutex + held. That routine used MAX_SCHEDULE_TIMEOUT. If an input needed completion + service taking the same mutex, the event worker could deadlock. Replay now + checks statuses without sleeping and fails an unexpectedly unmet dependency. + Initial publication waits off-lock in 10 ms cancellation/status-check slices, + with one absolute 2 s budget (10 s for a selected long submission). +* G17 mailbox receive drained an indefinitely replenished FIFO under rx_lock + with IRQs disabled. G17 now uses a oneshot threaded receive IRQ, consumes at + most 64 messages per pass, then sleeps 1-2 ms off-lock with the IRQ masked + when its budget is exhausted. The level condition retriggers without + discarding messages. Explicit receive polling has the same finite budget. +* G17 mailbox transmit without a send-empty IRQ could busy-poll for 500 ms + under tx_lock with IRQs disabled. Non-atomic G17 callers now sleep off-lock + between checks; an atomic G17 send polls for at most 1 ms. All sleeping + retries share a single 500 ms deadline, including contention between senders. + Removal, cookie, provider lifecycle and PM ownership gates are retained. +* Scheduler failure callbacks no longer synchronously join preparation leases + and RTKit receive work. They mark the runtime lost and queue the event worker. + Recovery joins preparation for at most 2 s. Runtime stop halts both firmware + producers first and joins preparation for at most 2 s before detachment. + If a builder remains stalled, its stopped firmware, graphs and reservations + remain owned in Shared::state. Timeout is never an ownership witness. That + path does not re-arm a polling loop; a later queued completion/owner return + may finish the stopped teardown. Explicit device shutdown still joins every + borrower before returning or releasing provider resources. +* Borrowed render graphs and off-lock queued retirements previously renewed + the watchdog without incrementing a render deferral counter. Both engines + now charge deferrals, and render transaction grace ends after eight polls. + Existing three replay credits remain finite and are never reset here. +* A per-device burst limit stops the runtime on the eighth actual scheduler + timeout in a 30 s window or the eighth restart handshake in a 30 s window. + VM changes, new publications, recovery generations and replay do not reset + those counters; terminal loss latches. The 30 s window contains eight + two-second expirations, including a stalled deferred timeout that cannot + complete its normal per-client settlement. This limits host recovery load even + when each individual handshake succeeds. Legitimate long jobs only consume + this budget after their independent bounded execution allowance expires. + +Lock order remains stop -> state and state -> preparation wait/reclaim. All +preparation joins and deferred fence completion/destruction run outside state. +Runtime stop uses try_lock for the stop owner, avoiding a recursive join from +an off-lock fence callback. Receive callbacks only retain/copy ingress and +queue work; no firmware ring is polled while an IRQ spinlock is held without +a finite iteration/time bound on the G17 paths. RX acknowledgement is still +level-sensitive, and TX publication/removal remains serialized by tx_lock. + +The existing control backpressure timer has 1,000 finite retries; its event +probes do not consume or reset that budget. Event-ring ingress is bounded by +EVENT_SLOTS; trace/statistics consume a snapshot producer. Render retirement +loops remove one retained tracker per successful iteration (or mark one as +accounted); compute replay consumes a retained replay per iteration. Shared +capacity deadlines remain absolute and failed fences do not release mappings +or return physical reservations. The new storm limits additionally bound +repeated recoveries and timeouts across those paths. + +KUnit exercises 64 consecutive watchdog timeouts with recovery pending under +real state/preparation mutexes, a retained lease that must time out rather +than be freed, late lease return with failed admission still closed, restart +storms and nonblocking replay status checks. The mailbox tests use normal RAM +as never-empty RX and never-draining TX registers, invoking the production +poll/send functions for 64 RX passes and eight atomic TX timeouts plus a +sleeping TX timeout/removal control. The isolated Asahi scenario runs with +lockdep, PROVE_LOCKING and panic_on_warn. Software tests cannot establish +that physical fabric/firmware or the original reset is fixed; separately +controller-gated Neo qualification remains necessary. diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5abd494eeb1f5..0bc1d5024566e 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -17,6 +17,14 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// A replay's original host inputs must still be signalled. Treat an unmet +/// input as a failed replay, never as a wait under the device mutex. +pub(crate) fn replay_status(status: i32) -> Result { + if status == 0 { Err(EBUSY) } + else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else { Ok(()) } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 27986b60d47b2..0588539c1ff75 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -143,23 +143,40 @@ impl Packet { } } + /// Replay holds the device mutex. Its original inputs were satisfied before + /// first publication; never sleep on a fence that needs event service here. + pub(crate) fn check_replay_dependencies(&self) -> Result { + for fence in &self.inputs { + super::dependency::replay_status(fence_status(fence))?; + } + for fence in &self.firmware { check_status(fence)?; } + Ok(()) + } + /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { + let polls = if self.command.long_job() { 5 } else { 1 }; + let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); + loop { + if self.cancel_requested() || self.completion.status().get() != 0 { + return Err(ECANCELED); + } + if fence_status(fence) != 0 { break; } + if super::now_ns() >= deadline { return Err(ECANCELED); } + // SAFETY: The retained fence owns the pointer throughout the bounded wait. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::time::msecs_to_jiffies(10) as isize, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited > 0 { break; } } check_status(fence)?; } @@ -311,7 +328,7 @@ impl sched::JobImpl for Job { return true; } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { + if deferred { job.victim_renewals += 1; } deferred diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index b00a5bb6b6191..0297c619b4f93 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -178,6 +178,17 @@ impl Gate { self.changed.notify_all(); } + /// Runtime failure/recovery must not join an indefinitely stalled builder. + /// Timeout retains every graph and reservation; it never authorizes freeing. + pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + if super::now_ns() >= deadline { return Err(ETIMEDOUT); } + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); + } + Ok(()) + } + /// Join builders with no device, queue or mapping lock held. pub(super) fn wait_drained(&self) { let mut wait = self.wait.lock(); diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ba368341d84d2..d5495b8c24f74 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,6 +9,7 @@ mod attribution; mod slot; +mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -99,6 +100,8 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, + timeouts: safety::Storm, + restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -112,8 +115,12 @@ impl State { Self { accepted: 0, pending: None, + timeouts: safety::Storm::new(), + restarts: safety::Storm::new(), } } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs new file mode 100644 index 0000000000000..c30e45e22579a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fail closed on repeated watchdog/restart traffic, even across client loss. + +const WINDOW_NS: u64 = 30_000_000_000; +const LIMIT: u8 = 8; + +pub(super) struct Storm { + start: Option, + count: u8, + stopped: bool, +} +impl Storm { + pub(super) const fn new() -> Self { + Self { start: None, count: 0, stopped: false } + } + pub(super) fn note(&mut self, now: u64) -> bool { + if self.stopped { return true; } + if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { + self.start = Some(now); + self.count = 0; + } + self.count += 1; + self.stopped = self.count >= LIMIT; + self.stopped + } +} + +#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] +mod tests { + use super::*; + use crate::g17::{preparation::Gate, recovery::State}; + use kernel::prelude::*; + + #[test] + fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { + let gate = Gate::new().unwrap(); + let lease = gate.try_enter().unwrap(); + let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); + gate.close(); + for n in 0..64 { + let stopped = { + let mut state = state.lock(); + state.request(0, -1).unwrap(); + assert!(state.pending()); + let stopped = state.timeout_limit(n * 2_000_000_000); + if stopped { gate.fail(); } // production state -> gate order + stopped + }; + assert_eq!(stopped, n >= u64::from(LIMIT - 1)); + } + // Recovery/stop must release state before joining the retained builder. + assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), + Err(error) if error == ETIMEDOUT)); + assert!(!gate.reopen()); // failure never gives back an owned reservation + drop(lease); + gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); + assert!(!gate.reopen()); + } + #[test] + fn successful_handshakes_do_not_reset_a_restart_storm() { + let mut state = State::new(); + for n in 0..64 { + assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); + } + assert!(state.restart_limit(u64::MAX)); + } + #[test] + fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } + for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } + assert!(storm.note(64 * WINDOW_NS)); + for _ in 0..1024 { assert!(storm.note(u64::MAX)); } + } + #[test] + fn replay_dependency_checks_never_wait_for_event_service() { + use crate::g17::dependency::replay_status; + assert_eq!(replay_status(0), Err(EBUSY)); + assert_eq!(replay_status(1), Ok(())); + assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e0565bf83db83..d51c586fa43dc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1379,6 +1379,38 @@ impl super::Firmware { } impl super::Shared { + /// Charge only real watchdog expirations, after the per-job allowance. A + /// changing client or recovery generation cannot mint a fresh storm budget. + pub(super) fn watchdog_timeout(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { return true; } + let stop = { + let mut state = self.state.lock(); + state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) + }; + if stop { self.lose_device_quietly(); } + stop + } + + /// Workload-triggered stop: halt producers first, then join only for a + /// bounded interval. A stalled borrower leaves stopped firmware and all its + /// mappings owned in state. Only a later joined stop may detach/free them. + pub(super) fn stop_runtime(&self) { + let Some(_stopping) = self.stop.try_lock() else { return; }; + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + firmware.secondary.stop(); + firmware.primary.stop(); + } + } + if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { + dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); + return; + } + let firmware = self.state.lock().take(); + if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } + } + /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 61c495e93a594..822e1d9d07713 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -18,8 +18,9 @@ impl Backend { } fn stop_after_error(&self) { + // Scheduler/fence callbacks never synchronously join builders or RTKit + // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - self.shared.stop_queues(); } fn rollback_unpublished_compute(&self) -> Result { @@ -190,6 +191,7 @@ impl job::Backend for Backend { fn timed_out(&self, packet: Arc) -> sched::Status { self.release_backlog(&packet); + if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -219,6 +221,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } + if !crate::g17::timeout::transaction_grace(renewals) { return false; } let mut state = self.shared.state.lock(); (*state) .as_deref_mut() diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0d4a29e779b4a..d71f410c76fb4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -83,7 +83,7 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; + packet.check_replay_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 5db6e7ad0c4c0..f8fcf253b41a5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -263,6 +263,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -300,7 +301,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); + self.stop_runtime(); return; } let mut deferred = { @@ -324,7 +325,7 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); } } @@ -345,7 +346,7 @@ impl crate::g17::Shared { }; deferred.finish(); if recover { - self.preparations.wait_drained(); + self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c8001..9e59a854fe1fc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -152,7 +152,7 @@ impl crate::g17::Shared { drop(state); drop(epoch); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); if let Some(mut pair) = pair { pair.stopped(&mut |event| { event.finish(); diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index df6656070a373..c809e50b2762c 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,6 +6,10 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// A borrowed render graph or queued off-lock retirement is not evidence of +/// progress. Share the finite deferral bound with cross-VM attribution. +pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } + /// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. /// Native ray traversal and AS builds can exceed one poll (CTS stress). Four /// extra polls give it ten seconds, still one tenth of the legacy scheduler's @@ -104,6 +108,12 @@ mod tests { } assert_eq!(as_build(u32::MAX), None); } + #[test] + fn borrowed_render_and_queued_retirement_cannot_defer_forever() { + for count in 0..=u8::MAX { + assert_eq!(transaction_grace(count), count < MAX_RENEWALS); + } + } fn head(vm: usize, published: u64, finished: bool) -> Head { Head { vm, published, finished } } diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h new file mode 100644 index 0000000000000..33365864f167a --- /dev/null +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* CPU-only transport fault injection: normal RAM stands in for the registers. + * The receive FIFO never empties and the send FIFO never gains space. + */ +#include + +static void apple_g17_test_rx(struct apple_mbox *mbox, + struct apple_mbox_msg msg, void *cookie) +{ + unsigned int *count = cookie; + + (*count)++; +} + +static void apple_g17_rx_flood_bounded_test(struct kunit *test) +{ + const struct apple_mbox_hw hw = { + .i2a_control = 0, .i2a_recv0 = 8, .i2a_recv1 = 16, + .control_empty = BIT(17), + }; + struct apple_mbox mbox = { .hw = &hw, .rx = apple_g17_test_rx }; + unsigned int count = 0; + void *regs = kunit_kzalloc(test, 32, GFP_KERNEL); + int n; + + KUNIT_ASSERT_NOT_NULL(test, regs); + mbox.regs = (void __iomem *)regs; + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + mbox.cookie = &count; + spin_lock_init(&mbox.rx_lock); + for (n = 0; n < 64; n++) + KUNIT_ASSERT_EQ(test, apple_mbox_poll(&mbox), APPLE_G17_MBOX_RX_BUDGET); + KUNIT_EXPECT_EQ(test, count, 64U * APPLE_G17_MBOX_RX_BUDGET); +} + +static void apple_g17_tx_stall_bounded_test(struct kunit *test) +{ + const struct apple_mbox_hw hw = { .control_full = BIT(16) }; + struct apple_mbox mbox = { .hw = &hw, .irq_send_empty = -1 }; + struct apple_mbox_msg msg = { 0 }; + void *regs = kunit_kzalloc(test, 32, GFP_KERNEL); + int n; + + KUNIT_ASSERT_NOT_NULL(test, regs); + mbox.regs = (void __iomem *)regs; + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + writel(BIT(16), mbox.regs); + spin_lock_init(&mbox.tx_lock); + for (n = 0; n < 8; n++) + KUNIT_ASSERT_EQ(test, apple_mbox_send(&mbox, msg, true), -ETIMEDOUT); + /* The G17 no-IRQ, non-atomic branch must sleep off-lock and expire. */ + KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ETIMEDOUT); + mbox.removing = true; + KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ENODEV); +} + +static struct kunit_case apple_g17_stall_cases[] = { + KUNIT_CASE(apple_g17_rx_flood_bounded_test), + KUNIT_CASE(apple_g17_tx_stall_bounded_test), + { } +}; +static struct kunit_suite apple_g17_stall_suite = { + .name = "apple_mbox_g17_stall", + .test_cases = apple_g17_stall_cases, +}; +kunit_test_suite(apple_g17_stall_suite); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5be648a40fe03..b23b556a6ae77 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -112,6 +113,8 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_MBOX_MSG1_MSG GENMASK(31, 0) #define APPLE_MBOX_TX_TIMEOUT 500 +#define APPLE_G17_MBOX_RX_BUDGET 64 +#define APPLE_G17_MBOX_ATOMIC_TX_US 1000 struct apple_mbox_hw { unsigned int reg_offset; @@ -266,10 +269,17 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +static bool apple_mbox_is_g17(const struct apple_mbox *mbox) +{ + return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || + mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; +} + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { - unsigned long flags; + unsigned long flags, remaining, now; + unsigned long deadline = jiffies + msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT); int ret; u32 mbox_ctrl; long t; @@ -282,10 +292,32 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { + now = jiffies; + if (time_after_eq(now, deadline)) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ETIMEDOUT; + } + remaining = deadline - now; + if (apple_mbox_is_g17(mbox) && !atomic && mbox->irq_send_empty < 0) { + /* The GPU has no send-empty IRQ. Do not busy wait with IRQs + * off while dead firmware keeps its FIFO full. Recheck both + * the removal fence and the one absolute deadline on retry. + */ + spin_unlock_irqrestore(&mbox->tx_lock, flags); + usleep_range(1000, 2000); + spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + continue; + } if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, !(mbox_ctrl & mbox->hw->control_full), 100, + apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_ATOMIC_TX_US : APPLE_MBOX_TX_TIMEOUT * 1000); if (ret) { @@ -316,8 +348,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, spin_unlock_irqrestore(&mbox->tx_lock, flags); t = wait_for_completion_interruptible_timeout( - &mbox->tx_empty, - msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT)); + &mbox->tx_empty, remaining); if (t <= 0) { /* A late IRQ may already have masked this under tx_lock. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -374,13 +405,14 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) struct apple_mbox_msg msg; int ret = 0; u32 mbox_ctrl; + u32 budget = apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_RX_BUDGET : U32_MAX; /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - while (!(mbox_ctrl & mbox->hw->control_empty)) { + while (!(mbox_ctrl & mbox->hw->control_empty) && budget--) { msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); msg.msg1 = FIELD_GET( APPLE_MBOX_MSG1_MSG, @@ -410,11 +442,20 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; + unsigned long flags; + int consumed = 0; - spin_lock(&mbox->rx_lock); + spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) - apple_mbox_poll_locked(mbox); - spin_unlock(&mbox->rx_lock); + consumed = apple_mbox_poll_locked(mbox); + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep + * so a firmware flood cannot monopolize a CPU at IRQ-thread priority. + * The level condition retriggers after return; no message is discarded. + */ + if (apple_mbox_is_g17(mbox) && consumed == APPLE_G17_MBOX_RX_BUDGET) + usleep_range(1000, 2000); return IRQ_HANDLED; } @@ -740,9 +781,15 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!irqname) return -ENOMEM; - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); if (ret) return ret; @@ -910,3 +957,7 @@ module_platform_driver(apple_mbox_driver); MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple Mailbox driver"); + +#if IS_ENABLED(CONFIG_KUNIT) +#include "mailbox-g17-test.h" +#endif From c314dbb5bd4284697bd065db87eb038021b18125 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 16:52:28 +1100 Subject: [PATCH 11/11] drm/asahi: expire background pool-release and render-replay polls An exited compute client can leave a published pool release without a scheduler job to time it out. A parked render replay can likewise re-arm the one-millisecond worker forever while publication remains unavailable. Timestamp those exact transactions and stop the runtime after ten seconds without their witness/commit. Wakes, new clients and repeated scans cannot refresh either deadline. Keep all backing and reservations owned until the existing consumed-release or stopped-processor boundary. Exercise the absolute deadline and exhaustion in production-policy KUnit. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/asahi-g17-recovery-safety.rst | 6 ++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 14 ++++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 9 +++++++++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 4 ++++ .../gpu/drm/asahi/g17/runtime/render/replay.rs | 4 +++- drivers/gpu/drm/asahi/g17/timeout.rs | 17 +++++++++++++++++ tools/asahi/g17-growth-tests.py | 9 +++++++-- 8 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Documentation/gpu/asahi-g17-recovery-safety.rst b/Documentation/gpu/asahi-g17-recovery-safety.rst index f56d08ac2e570..37a5f0a9b0ab0 100644 --- a/Documentation/gpu/asahi-g17-recovery-safety.rst +++ b/Documentation/gpu/asahi-g17-recovery-safety.rst @@ -34,6 +34,12 @@ reproduction is claimed here. the watchdog without incrementing a render deferral counter. Both engines now charge deferrals, and render transaction grace ends after eight polls. Existing three replay credits remain finite and are never reset here. +* Idle compute-pool releases have no scheduler job once their client is lost. + Record a timestamp on each exact release publication; a missing consumer + witness after 10 s stops the runtime while retaining the backing. Parked + render replays similarly expire after 10 s rather than polling forever when + admission or publication cannot proceed. Wakes/new clients do not refresh + either transaction timestamp; exact completion clears it. * A per-device burst limit stops the runtime on the eighth actual scheduler timeout in a 30 s window or the eighth restart handshake in a 30 s window. VM changes, new publications, recovery generations and replay do not reset diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c17f737cdd507..6f9a0f5599d75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -477,6 +477,7 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -540,6 +541,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -653,6 +655,10 @@ impl Queue { matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) } + pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { + self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + } + /// A closed owner can still retain its last commands or their completion /// stamps. Admission may wait for retirement, without treating that close /// as permission to reuse any firmware-owned backing. @@ -1357,6 +1363,7 @@ impl Queue { ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); + self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1381,7 +1388,7 @@ impl Queue { let pages = self.pool.take_pages(); let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; + self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1400,6 +1407,7 @@ impl Queue { self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1419,7 +1427,8 @@ impl Queue { let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - } else { self.pool_phase = PoolPhase::Vacant; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) @@ -1445,6 +1454,7 @@ impl Queue { self.pool.replace_pages(pages)?; let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 22e3fb49b940f..be19596e39fa6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -171,6 +171,7 @@ pub(crate) struct Pair { context_update: bool, /// A retired, never-executed group awaiting republication on this pair. replay: Option>, + replay_since: Option, } impl Pair { #[inline(never)] @@ -258,6 +259,7 @@ impl Pair { previous: None, displaced: None, replay: None, + replay_since: None, context_update: true, })) } @@ -292,6 +294,9 @@ impl Pair { pub(crate) fn replay_polling(&self) -> bool { self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) } + pub(crate) fn replay_stalled(&self, now: u64) -> bool { + self.replay.is_some() && crate::g17::timeout::background_stalled(self.replay_since, now) + } pub(crate) fn replay_front(&self) -> Option> { self.replay.clone() } @@ -300,6 +305,7 @@ impl Pair { return Err(EIO); } self.replay = None; + self.replay_since = None; Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -1233,6 +1239,7 @@ impl Pair { self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); self.replay = Some(packet); + self.replay_since = Some(now); Ok(true) } @@ -1247,6 +1254,7 @@ impl Pair { let Some(packet) = self.replay.take() else { return Ok(()); }; + self.replay_since = None; defer(Deferred::Retired(packet.completion.clone(), Err(error))) } @@ -1434,6 +1442,7 @@ impl Pair { self.active.pop().ok_or(EIO)?; } if let Some(packet) = self.replay.take() { + self.replay_since = None; defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 034da4dd2eeea..550a9ce5ed866 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -228,6 +228,7 @@ impl ParkedPair { previous: self.previous.take(), displaced: None, replay: None, + replay_since: None, context_update: true, })) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d51c586fa43dc..4750d1f2e4438 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1471,6 +1471,10 @@ impl super::Firmware { } } self.queues.observe_pool_releases(&self.init, deferred)?; + if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_stalled(now)) { + return Err(ETIMEDOUT); + } if self.render_control_backpressured() { return Ok(None); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index ce9771778acd4..bf5ba6caa7a79 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -27,7 +27,7 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - let Some((packet, owner, qids, closed, busy)) = + let Some((packet, owner, qids, closed, busy, stalled)) = self.queues.render.entries[slot].as_ref().and_then(|entry| { let pair = entry.pair.as_ref()?; let packet = pair.replay_front()?; @@ -40,11 +40,13 @@ impl Firmware { || entry.growing || entry.pending_timeout || entry.pending_error.is_some(), + pair.replay_stalled(crate::g17::now_ns()), )) }) else { continue; }; + if stalled { return Err(ETIMEDOUT); } if busy { continue; } diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index c809e50b2762c..5dc04ef5d09a0 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -18,6 +18,12 @@ pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS pub(crate) const POLL_MS: u32 = 2000; const EXECUTION_RENEWALS: u8 = 4; +/// A background control/replay poll has no scheduler fence to expire it. +/// Its timestamp belongs to the exact transaction and is never renewed by work. +pub(crate) fn background_stalled(since: Option, now: u64) -> bool { + since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) +} + /// Input comes only from fully validated commands. Render metadata covers the /// VDM half (vertex/tessellation/geometry) and the fragment half independently. /// AS builders do not traverse rays and need an explicit, bounded UAPI hint. @@ -114,6 +120,17 @@ mod tests { assert_eq!(transaction_grace(count), count < MAX_RENEWALS); } } + #[test] + fn background_control_and_replay_polls_have_an_absolute_deadline() { + let since = 42; + assert!(!background_stalled(None, u64::MAX)); + assert!(!background_stalled(Some(since), since - 1)); + assert!(!background_stalled(Some(since), since + 9_999_999_999)); + for _ in 0..1024 { + assert!(background_stalled(Some(since), since + 10_000_000_000)); + } + assert!(background_stalled(Some(since), u64::MAX)); + } fn head(vm: usize, published: u64, finished: bool) -> Head { Head { vm, published, finished } } diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index e97e29d248e2c..1236f166876af 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -33,12 +33,12 @@ def fixture(): impl Graph {fn set_owner(&mut self,_:&Context)->Result{Ok(())}} struct Queue { owner:Option,installed:bool,growing:bool,growth_blocks:usize,free_list_generation:u64, - pool_phase:PoolPhase,pool:Pool,binding:Option,previous:Option,graph:Graph, + pool_phase:PoolPhase,release_sent_ns:Option,pool:Pool,binding:Option,previous:Option,graph:Graph, active:Fifo,replays:Fifo,quarantined:bool,released:bool,retirement_ready:bool, retired_by_teardown:bool,released_at:u64,exit:Option<()>,needs_fresh_backing:bool, } fn idle(live:&Arc)->Queue{Queue{owner:Some(1),installed:true,growing:false,growth_blocks:0, - free_list_generation:0,pool_phase:PoolPhase::Populated,pool:Pool{pages:Some(Pages(10)),fail:false}, + free_list_generation:0,pool_phase:PoolPhase::Populated,release_sent_ns:None,pool:Pool{pages:Some(Pages(10)),fail:false}, binding:Some(Binding{context:Context::new(live),pool_alias:Some(1),descriptors:Some(2)}),previous:None,graph:Graph, active:Fifo{len:0},replays:Fifo{len:0},quarantined:false,released:false,retirement_ready:true, retired_by_teardown:false,released_at:0,exit:None,needs_fresh_backing:false}} @@ -55,6 +55,7 @@ def fixture(): impl Pages {fn page_list_va(&self)->u64{self.0}} mod mmu {pub type KernelMapping=u64;} mod g17 { + pub fn now_ns()->u64{crate::now_ns()} pub mod freelist {pub(crate) use crate::Pages; pub fn control_consumed(c:u32,_:u32,t:u32)->crate::Result{Ok(c==t)}} pub mod initdata { pub struct InitData {pub fail:bool} @@ -101,6 +102,7 @@ def fixture(): let mut q=idle(&live);q.retirement_ready=true; assert!(q.begin_growth(&init,40).unwrap());let token=q.growth_token().unwrap(); assert!(q.growing);assert!(q.pool_phase==PoolPhase::ReleaseSent(3)); + assert_eq!(q.release_sent_ns,Some(now_ns()),"exact publication starts the release deadline"); assert_eq!(q.abort_growth(Growth{qid:8,phase:token.phase}).unwrap(),None,"wrong physical queue"); assert_eq!(q.abort_growth(Growth{qid:token.qid,phase:PoolPhase::ReleaseSent(4)}).unwrap(),None,"stale release transaction"); assert!(q.growing); @@ -113,6 +115,7 @@ def fixture(): assert!(!q.growing);assert!(q.owner.is_none());assert_eq!(q.growth_blocks,0); assert_eq!(q.abort_growth(token).unwrap(),None); assert!(q.pool.pages.is_some(),"abort must retain backing"); + assert_eq!(q.release_sent_ns,Some(now_ns()),"abort must not refresh or discard an owned deadline"); assert!(!q.reusable(),"published release still owns backing"); let mut registry=Registry{compute:vec![Some(Entry{queue:Some(Box::new(q))})]}; let mut deferred=DeferredBatch{vacated:vec![],capacity:0}; @@ -124,6 +127,7 @@ def fixture(): if !consumed_first {assert_eq!(registry.observe_pool_releases(&init,&mut deferred).unwrap(),1);} let q=registry.compute[0].as_ref().unwrap().queue.as_ref().unwrap(); assert!(q.pool_phase==PoolPhase::Vacant);assert!(q.reusable()); + assert_eq!(q.release_sent_ns,None,"only the consumed release clears its deadline"); assert_eq!(deferred.vacated[0]._pages.as_ref().unwrap().0,10); drop(registry);drop(deferred);assert_eq!(live.load(Ordering::SeqCst),0); }} @@ -141,6 +145,7 @@ def fixture(): let mut q=idle(&live);q.retirement_ready=true;q.begin_growth(&init,40).unwrap(); let token=q.growth_token().unwrap();let mut pages=Some(Pages(20));let mut alias=Some(30); let old=q.finish_growth(&init,&mut pages,&mut alias,40).unwrap(); + assert_eq!(q.release_sent_ns,None,"committed replacement clears the witnessed release deadline"); assert_eq!(old._pages.unwrap().0,10);assert_eq!(old._alias,Some(1));assert!(pages.is_none());assert!(alias.is_none()); assert_eq!(q.pool.pages.as_ref().unwrap().0,20);assert_eq!(q.abort_growth(token).unwrap(),None,"committed growth cannot be aborted"); println!("PASS production growth: abort before/after consumption and close; stale witness/collector preserve backing; failed row/header rearm preserves both owners; commit invalidates abort token");