diff --git a/drivers/base/base.h b/drivers/base/base.h index a5b7abc10ff02..d16026fdb2ef0 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -222,6 +222,8 @@ typedef void (*dr_node_free_t)(struct devres_node *node); struct devres_node { struct list_head entry; + /* Device-list membership, cleared under devres_lock before detachment. */ + struct device *owner; dr_node_release_t release; dr_node_free_t free_node; const char *name; diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 9d9842fc5a19f..26e5518b6482b 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -41,6 +41,7 @@ void devres_node_init(struct devres_node *node, dr_node_free_t free_node) { INIT_LIST_HEAD(&node->entry); + WRITE_ONCE(node->owner, NULL); node->release = release; node->free_node = free_node; } @@ -156,6 +157,7 @@ static void add_dr(struct device *dev, struct devres_node *node) devres_log(dev, node, "ADD"); BUG_ON(!list_empty(&node->entry)); list_add_tail(&node->entry, &dev->devres_head); + WRITE_ONCE(node->owner, dev); } static void replace_dr(struct device *dev, @@ -164,6 +166,8 @@ static void replace_dr(struct device *dev, devres_log(dev, old, "REPLACE"); BUG_ON(!list_empty(&new->entry)); list_replace(&old->entry, &new->entry); + WRITE_ONCE(old->owner, NULL); + WRITE_ONCE(new->owner, dev); } /** @@ -367,18 +371,14 @@ EXPORT_SYMBOL_GPL(devres_get); bool devres_node_remove(struct device *dev, struct devres_node *node) { - struct devres_node *__node; - guard(spinlock_irqsave)(&dev->devres_lock); - list_for_each_entry_reverse(__node, &dev->devres_head, entry) { - if (__node == node) { - list_del_init(&node->entry); - devres_log(dev, node, "REM"); - return true; - } - } - - return false; + /* A release-all/group pass may own a detached node on its private list. */ + if (READ_ONCE(node->owner) != dev) + return false; + list_del_init(&node->entry); + WRITE_ONCE(node->owner, NULL); + devres_log(dev, node, "REM"); + return true; } /** @@ -405,6 +405,7 @@ void *devres_remove(struct device *dev, dr_release_t release, dr = find_dr(dev, release, match, match_data); if (dr) { list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr->data; } @@ -500,6 +501,7 @@ static int remove_nodes(struct device *dev, if (&node->entry == first) first = first->next; list_move_tail(&node->entry, todo); + WRITE_ONCE(node->owner, NULL); cnt++; } } @@ -530,7 +532,9 @@ static int remove_nodes(struct device *dev, * nodes are always before both. */ list_move_tail(&grp->node[0].entry, todo); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); } } @@ -685,7 +689,9 @@ void devres_remove_group(struct device *dev, void *id) grp = find_group(dev, id); if (grp) { list_del_init(&grp->node[0].entry); + WRITE_ONCE(grp->node[0].owner, NULL); list_del_init(&grp->node[1].entry); + WRITE_ONCE(grp->node[1].owner, NULL); devres_log(dev, &grp->node[0], "REM"); } else WARN_ON(1); @@ -849,6 +855,7 @@ static struct devres_action *remove_action(struct device *dev, return ERR_PTR(-ENOENT); list_del_init(&dr->node.entry); + WRITE_ONCE(dr->node.owner, NULL); devres_log(dev, &dr->node, "REM"); return dr; diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index db2561d4d062f..bcbbb2e7e875d 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -24,9 +24,10 @@ pub(super) enum Site { LowerVa, UpperVa, AliasVa, + ExecutionContexts, } -const SITE_COUNT: usize = Site::AliasVa as usize + 1; +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; /// First and power-of-two refusals per site keep a pressure storm bounded in @@ -78,7 +79,7 @@ impl kernel::debugfs::Writer for View { Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, - Site::LowerVa, Site::UpperVa, Site::AliasVa, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, ].iter().zip(REFUSALS.iter()) { writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; } diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 13fb35d03d04d..ea58cfb8de286 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,81 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +249,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -220,7 +296,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -282,14 +358,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -339,7 +415,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -405,6 +481,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index fe874992cffeb..c93ab25d55ce3 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -194,12 +196,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 49544300531a8..3b31c3cead064 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -69,19 +69,81 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; Self::from_backing(backing, blocks, layout) } fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { @@ -147,13 +209,16 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let (backing, [global, client]) = alloc.lower_with_aliases( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - [render_global, client], - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 0588539c1ff75..575a9a7adc976 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -196,6 +196,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index a2f8094dfb996..7aaca95992f8d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -291,10 +298,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -330,6 +421,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -364,14 +458,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -399,14 +524,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -416,19 +553,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -441,7 +578,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9129418f4a16f..04dd027a975f7 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -480,6 +480,8 @@ struct Firmware { registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -981,6 +983,7 @@ impl Gpu { registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 0165a00568118..7fc4d79a5f95c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -372,6 +372,10 @@ impl Pool { GFP_KERNEL, ) } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9a105bf9da03f..9ccdd703106b7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -589,6 +589,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index be19596e39fa6..b69cd4d565736 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -172,6 +172,12 @@ pub(crate) struct Pair { /// A retired, never-executed group awaiting republication on this pair. replay: Option>, replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] @@ -182,6 +188,7 @@ impl Pair { pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -219,8 +226,9 @@ impl Pair { } let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -234,7 +242,7 @@ impl Pair { word, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -261,6 +269,8 @@ impl Pair { replay: None, replay_since: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -798,6 +808,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -1596,6 +1611,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.is_some() || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 550a9ce5ed866..5b3f2a00bad6d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -31,6 +31,11 @@ pub(crate) struct ParkedPair { registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -91,6 +96,7 @@ impl Pair { registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -105,6 +111,13 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } @@ -230,6 +243,8 @@ impl ParkedPair { replay: None, replay_since: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4750d1f2e4438..a9622d5909499 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -136,6 +136,7 @@ impl Registry { queue.share_idle()?; } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -144,6 +145,12 @@ impl Registry { .filter(|queue| queue.reusable()) { admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -174,6 +181,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -181,6 +189,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -431,24 +443,54 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -1082,6 +1124,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, @@ -1261,6 +1312,7 @@ impl Backend { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; + let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1277,7 +1329,22 @@ impl Backend { Backend::ensure_compute(self, scratch.pages(), &admission)?; admission.check()?; let lease = self.shared.preparations.enter_until(&admission)?; - packet.completion.work_state()?.acquire()?; + // Work states free as this context's earlier commands retire. + // Wait for one under the packet's admission deadline, as render + // publication does, instead of failing the packet. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + admission.check()?; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { @@ -1478,9 +1545,16 @@ impl super::Firmware { if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 822e1d9d07713..0d77ef4a54476 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -190,6 +190,15 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34ae..80f88d09191b4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -50,7 +50,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 90ca9ed91c63a..88340442bbcae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -46,7 +46,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -130,6 +136,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -163,6 +177,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -182,6 +197,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -341,7 +357,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -872,9 +890,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 856d6a0501548..e3cb174721aaa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,7 +271,10 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 9e59a854fe1fc..4ede20d316c64 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -163,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 411b4f0fd962f..2acd464bbeea4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,6 +29,7 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), @@ -76,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -137,6 +145,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -168,6 +183,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -246,6 +263,7 @@ impl super::super::Backend { Pair::new( alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -401,6 +419,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -467,6 +488,10 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { @@ -503,6 +528,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -560,8 +589,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -576,8 +605,8 @@ impl super::super::Backend { // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling.slot(), packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -589,8 +618,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -666,6 +696,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index b63c3dc5260ef..5db82a08bc8d1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index a27e2222955dd..b77fc13b7ed2f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,29 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -30,6 +52,7 @@ impl Backend { &self, packet: &Arc, mut next: Option, + sleep: bool, ) -> Result { let pool = { let state = self.shared.state.lock(); @@ -49,6 +72,10 @@ impl Backend { let lease = if let Some(lease) = next.take() { lease } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { // Atomics only: the callback runs under the pool mutex. if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { @@ -70,7 +97,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started)?; + self.wait_render_capacity(started, sleep)?; } drop(reclaimed); continue; @@ -104,7 +131,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - self.wait_render_capacity(started)?; + self.wait_render_capacity(started, sleep)?; } drop(reclaimed); } @@ -113,10 +140,30 @@ impl Backend { } } + /// A pair is borrowed, still settling, or waiting for the release of the + /// scheduler it last ran for to be consumed and the pair parked. The event + /// worker observes both, so kick it on every poll; time out only after the + /// same bound as a capacity wait. + fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + /// A closing owner may still hold its pair until exact retirement. Wait /// outside both the device mutex and preparation gate so the event worker /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64) -> Result { + fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } self.wait_capacity(started) } @@ -135,12 +182,15 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early)?; + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); loop { @@ -155,7 +205,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -166,6 +220,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { + if !sleep { + return Err(FALLBACK); + } if state_polls == 1000 { return Err(ETIMEDOUT); } @@ -178,25 +235,21 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, sleep) { Err(error) if error == ENOSPC => { - self.wait_capacity(capacity_started)?; + self.wait_render_capacity(capacity_started, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_capacity(capacity_started)?; + self.wait_render_capacity(capacity_started, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { @@ -207,6 +260,9 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { + if !sleep { + return Err(FALLBACK); + } if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { return Err(ETIMEDOUT); } @@ -214,11 +270,9 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -261,6 +315,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -394,6 +458,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0..eb04f80361955 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01..f8020e8962f84 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b..53518a76e91a4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 41f04be56e28d..62282b2635b8a 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -273,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py index b551cbaa96339..eca8b73c5b12c 100644 --- a/tools/asahi/g17-compute-deadline-tests.py +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -52,8 +52,8 @@ def fixture(): fn advance(ns:u64){NOW.fetch_add(ns,Ordering::SeqCst);} mod kernel{pub mod time{ pub fn msecs_to_jiffies(ms:u32)->u32{ms} - pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms)}} - pub mod delay{pub fn fsleep(d:super::Delta){crate::advance(d.0*1_000_000);}} + pub struct Delta(pub u64);impl Delta{pub fn from_millis(ms:u64)->Self{Self(ms*1_000_000)}pub fn from_micros(us:i64)->Self{Self(us as u64*1_000)}} + pub mod delay{pub fn fsleep(d:super::Delta){crate::advance(d.0);}} }} ''' + admission + r''' struct Wait; @@ -70,7 +70,7 @@ def fixture(): struct Scratch;impl Scratch{fn pages(&self)->u16{0}} enum Validated{Compute{scratch:Scratch}} struct Status;impl Status{fn get(&self)->i32{if now_ns()>=STATUS.load(Ordering::SeqCst){-5}else{0}}} -struct Work;impl Work{fn acquire(&self)->Result{Ok(())}} +struct Work;impl Work{fn acquire(&self)->Result{if MODE.load(Ordering::SeqCst)==4{Err(EBUSY)}else{Ok(())}}} struct Completion;impl Completion{fn status(&self)->Status{Status}fn work_state(&self)->Result{Ok(Work)}} struct Packet{command:Validated,completion:Completion} impl Packet{fn check_dependencies(&self)->Result{Ok(())}fn cancel_requested(&self)->bool{now_ns()>=CANCEL.load(Ordering::SeqCst)}fn is_published(&self)->bool{false}} @@ -109,6 +109,13 @@ def fixture(): if mode==0{assert_eq!(CALLS.load(Ordering::SeqCst),4);assert_eq!(now_ns(),WORK_RING_WAIT_NS);} println!("PASS mode={} result={:?} calls={} elapsed_ms={}",mode,r,CALLS.load(Ordering::SeqCst),now_ns()/1_000_000); } + // A context without a free work state waits for one under the admission + // deadline instead of failing the packet at once. + MODE.store(4,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); + let r=b.compute_packet(&p,&mut DeferredBatch); + assert_eq!(r,Err(EBUSY),"a busy work state ends at the admission deadline"); + assert!(now_ns()>=WORK_RING_WAIT_NS&&now_ns()<=WORK_RING_WAIT_NS+10_000_000,"a busy work state waits for the admission deadline"); + println!("PASS busy work state waits elapsed_ms={}",now_ns()/1_000_000); for status in [false,true] { MODE.store(2,Ordering::SeqCst);NOW.store(0,Ordering::SeqCst);CALLS.store(0,Ordering::SeqCst); if status{STATUS.store(10_000_000,Ordering::SeqCst);}else{CANCEL.store(10_000_000,Ordering::SeqCst);} diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py new file mode 100755 index 0000000000000..201f525718950 --- /dev/null +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the ordering between physical-queue owner changes and scheduler releases. + +The production release table, pair switch tracking, release readiness and compute +switch predicate are compiled against host stubs. This is not GPU execution. The +captured firmware crash (a render pair rebound in place and kicked by a same-VM +successor while its previous scheduler's release was published and the pair not +yet parked) must fail the same assertions once either rule is removed. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + opening = source.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def ordered(source, *needles): + """Every needle occurs, in this order, inside `source`.""" + position = 0 + for needle in needles: + found = source.find(needle, position) + assert found >= 0, f'{needle!r} missing or out of order' + position = found + len(needle) + + +def check_guards(): + prepare = (ROOT / 'runtime/render/prepare.rs').read_text() + build = item(prepare, 'fn build_render_pair(') + # The in-place rebind is refused while the pair's scheduler release is pending. + ordered(build, 'if !pair.reusable()?', 'firmware.queues.teardown.releasing(pair.context())', + 'return Err(EBUSY);', 'Build::Rebind {') + rebind = item(prepare, 'fn rebind_pair(') + # A live donor that closed while the binding was prepared is caught at commit. + ordered(rebind, 'firmware.queues.teardown.releasing(pair.context())', 'return Err(EBUSY);', + 'firmware.queues.teardown.displace(', 'pair.bind_owner(') + submit = (ROOT / 'runtime/render/submit.rs').read_text() + publish = item(submit, 'pub(super) fn publish_render(') + ordered(publish, '.owner_switch_from(&packet.context)', 'releasing_scheduler(scheduler)', + 'return Err(EBUSY);', 'acquire_owner(buffer_id)', 'pair.publish(') + render = (ROOT / 'queue/render.rs').read_text() + pair_publish = item(render, 'pub(crate) fn publish(') + # The switch is recorded only after the outer publication succeeded. + ordered(pair_publish, 'let outer_next = match result', 'self.ordinal = next_ordinal;', + 'self.switch = Some((self.executed, ordinal));', 'self.executed = scheduler;', + 'self.announce_publication(') + parked = (ROOT / 'queue/render/parked.rs').read_text() + # Parking keeps the last scheduler; only a consumed release clears it. + ordered(item(parked, 'pub(crate) fn park('), 'executed: self.executed,') + ordered(item(parked, 'pub(crate) fn commit('), 'executed: self.executed,', 'switch: None,') + detach = item((ROOT / 'runtime/render/teardown.rs').read_text(), 'fn detach_context(') + ordered(detach, '.park();', 'parked.release_consumed(context.scheduler_va());') + runtime = (ROOT / 'runtime.rs').read_text() + # A compute queue does not change owner while its scheduler is released. + ordered(item(runtime, 'pub(super) fn rebind_compute('), '.filter(|queue| queue.reusable())', + 'teardown.releasing(bound)', 'continue;', 'queue.bind_owner(owner, context.clone())?;') + ordered(item(runtime, 'pub(super) fn bind_rearmed('), 'teardown.releasing(bound)', 'return Err(EAGAIN);', + 'queue.bind_owner(') + ordered(item(runtime, 'fn publish_compute('), '.pending_switch_from()', 'queues.teardown.releasing(previous)', + 'return Err(EAGAIN);', 'queue.publish(') + # A busy pair (for example one whose scheduler release is still being consumed) + # kicks the event worker, which observes the release and parks the pair. + ordered(item(submit, 'fn wait_render_busy('), 'WORK_RING_WAIT_NS', 'return Err(ETIMEDOUT);', + 'self.shared.queue_events();', 'fsleep(') + for name, path in [('take_pair', 'runtime/render.rs'), ('take_render_observation', 'runtime/render/observe.rs')]: + body = item((ROOT / path).read_text(), f'fn {name}(') + ordered(body, 'entry.borrowed_switch = pair.switch_in_flight();', 'entry.pair.take()') + print('PASS guard placement: rebind decision, rebind commit, first publication, switch record, borrows, parked scheduler, compute owner changes, busy wait') + + +def fixture(): + teardown = (ROOT / 'teardown.rs').read_text() + assert ('#[derive(Clone, Copy, PartialEq, Eq)]\nenum Phase {\n Deferred,\n Draining,\n' + ' Released(u32),\n}') in teardown + pending = '\n'.join(item(teardown, name) for name in [ + 'pub(crate) fn displace(', 'pub(crate) fn close(', 'fn insert(', 'pub(crate) fn releasing(', + 'pub(crate) fn releasing_scheduler(', 'fn find(']).replace('pub(crate) ', '') + pending = pending.replace('self.entries.push(entry, GFP_KERNEL)?;', 'self.entries.push(entry);') + render = (ROOT / 'queue/render.rs').read_text() + switch_methods = '\n'.join(item(render, name) for name in [ + 'pub(crate) fn owner_switch_from(', 'pub(crate) fn switch_in_flight(']).replace('pub(crate) ', '') + record = render[render.index(' let scheduler = packet.context.scheduler_va();'): + render.index(' self.executed = scheduler;') + len(' self.executed = scheduler;')] + idle = item((ROOT / 'runtime/render/teardown.rs').read_text(), + 'pub(in crate::g17::runtime) fn teardown_idle(').replace('pub(in crate::g17::runtime) ', '') + compute_switch = item((ROOT / 'queue/compute.rs').read_text(), + 'pub(crate) fn switching_from(').replace('pub(crate) ', '') + compute_idle = item((ROOT / 'runtime/teardown.rs').read_text(), 'fn compute_idle(') + return r''' +#![allow(dead_code)] +use std::sync::Arc; +type Error = i32; +type Result = std::result::Result; +struct Context { scheduler: u64, published: bool } +impl Context { + fn scheduler_va(&self) -> u64 { self.scheduler } + fn is_published(&self) -> bool { self.published } +} +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { Deferred, Draining, Released(u32) } +struct Entry { context: Arc, phase: Phase, armed: u64, announce_pending: bool } +struct Pending { entries: Vec } +impl Pending { +PENDING +} +struct Ticket { ordinal: u64 } +struct Active { ticket: Ticket } +struct Packet { context: Arc } +struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Option<()>, active: Vec, idle: bool } +impl Pair { +SWITCH + /// Host stand-in for a successful publication of `ordinal` (stand-in, not production). + fn publish(&mut self, packet: &Packet, ordinal: u64) { +RECORD + self.active.push(Active { ticket: Ticket { ordinal } }); + } + fn idle_for_context(&self, _: &Arc) -> Result { Ok(self.idle) } +} +struct RenderEntry { + context: Option>, + preparing: Option<()>, + growing: bool, + pair: Option>, + borrowed_switch: Option, +} +struct RenderState { entries: Vec> } +impl RenderState { +IDLE +} +struct Previous { context: Arc, publication: Option<(u64, u64)> } +struct Queue { previous: Option, bound: Option> } +impl Queue { +COMPUTE_SWITCH + fn idle_for_context(&self, context: &Arc) -> bool { + !self.bound.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } +} +struct ComputeEntry { queue: Option> } +struct Service { compute: Vec> } +impl Service { +COMPUTE_IDLE +} +fn context(scheduler: u64) -> Arc { Arc::new(Context { scheduler, published: true }) } +fn main() { + let (b, c, d) = (context(0x1000), context(0x2000), context(0x3000)); + // Captured crash: GL context B closes, its release is published, then the same-VM + // successor C finds B's pairs still installed (not parked) and rebinds them in place. + let mut table = Pending { entries: Vec::new() }; + table.close(&b, 1).unwrap(); + assert!(table.releasing(&b), "armed release refuses an owner change"); + table.entries[0].phase = Phase::Released(67); + assert!(table.releasing(&b), "published, unconsumed release refuses an owner change"); + assert!(table.releasing_scheduler(0x1000) && !table.releasing_scheduler(0x2000)); + // Consumption and detach remove the entry; the pair is parked and reinstalled fresh. + table.entries.clear(); + assert!(!table.releasing(&b) && !table.releasing_scheduler(0x1000)); + // A displacement from a live owner is not a release; its later close arms one. + table.displace(&d, false, 2).unwrap(); + assert!(!table.releasing(&d), "a live displaced owner may still lose an idle pair"); + table.close(&d, 3).unwrap(); + assert!(table.releasing(&d), "the donor's close arms its release"); + // An unpublished context has no firmware scheduler state and no release. + let quiet = Arc::new(Context { scheduler: 0x4000, published: false }); + table.close(&quiet, 4).unwrap(); + assert!(!table.releasing(&quiet)); + + // Switch tracking across an owner change of one pair. + let mut pair = Pair { executed: 0, switch: None, replay: None, active: Vec::new(), idle: true }; + assert_eq!(pair.owner_switch_from(&c), None, "a fresh graph has no previous scheduler"); + pair.publish(&Packet { context: d.clone() }, 0); + pair.active.clear(); + assert_eq!(pair.owner_switch_from(&d), None, "the same owner never switches"); + assert_eq!(pair.owner_switch_from(&c), Some(0x3000)); + pair.publish(&Packet { context: c.clone() }, 1); + assert_eq!(pair.switch_in_flight(), Some(0x3000)); + let mut render = RenderState { entries: vec![Some(RenderEntry { + context: Some(c.clone()), preparing: None, growing: false, pair: Some(Box::new(pair)), borrowed_switch: None, + })] }; + assert!(!render.teardown_idle(&d).unwrap(), "release waits for the new owner's first command"); + assert!(render.teardown_idle(&b).unwrap(), "unrelated releases are not held"); + // A borrowed pair is judged by its snapshot; a stale snapshot outside a borrow is ignored. + let borrowed = render.entries[0].as_mut().unwrap(); + let pair = borrowed.pair.take().unwrap(); + borrowed.borrowed_switch = pair.switch_in_flight(); + borrowed.preparing = Some(()); + assert!(!render.teardown_idle(&d).unwrap(), "a borrowed switching pair still holds the release"); + let entry = render.entries[0].as_mut().unwrap(); + entry.preparing = None; + entry.pair = Some(pair); + entry.pair.as_mut().unwrap().active.clear(); + assert!(render.teardown_idle(&d).unwrap(), "retirement of the first command releases the hold"); + let entry = render.entries[0].as_mut().unwrap(); + entry.pair.as_mut().unwrap().active.push(Active { ticket: Ticket { ordinal: 1 } }); + entry.pair = None; + assert!(render.teardown_idle(&d).unwrap(), "a stale snapshot without a borrow holds nothing"); + + // Compute: only a published first command after the change holds the release. + let mut service = Service { compute: vec![Some(ComputeEntry { queue: Some(Box::new(Queue { + previous: Some(Previous { context: b.clone(), publication: None }), bound: Some(c.clone()), + })) })] }; + assert!(service.compute_idle(&b), "an unpublished rebind does not hold the release"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous.as_mut().unwrap().publication = Some((1, 0)); + assert!(!service.compute_idle(&b), "compute release waits for the new owner's first command"); + service.compute[0].as_mut().unwrap().queue.as_mut().unwrap().previous = None; + assert!(service.compute_idle(&b)); + assert!(!service.compute_idle(&c), "the bound owner still needs its own retirement"); + println!("PASS release table gate, pair switch hold, borrowed snapshot and compute switch hold"); +} +'''.replace('PENDING', pending).replace('SWITCH', switch_methods, 1).replace('RECORD', record) \ + .replace('IDLE', idle, 1).replace('COMPUTE_SWITCH', compute_switch).replace('COMPUTE_IDLE', compute_idle) + + +def main(): + check_guards() + code = fixture() + mutations = { + 'fixed': (None, None, None), + 'no-release-gate': ('.is_some_and(|index| self.entries[index].phase != Phase::Deferred)', + '.is_some_and(|_| false)', 'armed release refuses an owner change'), + 'no-render-hold': ('Some(pair) => pair.switch_in_flight() == scheduler,', 'Some(_) => false,', + "release waits for the new owner's first command"), + 'no-compute-hold': ('queue.idle_for_context(context) && !queue.switching_from(context)', + 'queue.idle_for_context(context)', + "compute release waits for the new owner's first command"), + } + with tempfile.TemporaryDirectory(prefix='g17-owner-release-') as directory: + root = Path(directory) + for name, (old, new, expected) in mutations.items(): + current = code + if old is not None: + assert current.count(old) == 1, name + current = current.replace(old, new) + path = root / (name + '.rs') + path.write_text(current) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path), '-o', str(binary)], + check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if old is None: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0 and expected in result.stderr, result.stderr + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 36d4ced3510bb..35f970799b086 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -232,6 +232,67 @@ def item(s,start): subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) subprocess.run([str(root/'unused-render-guard-controls')],check=True) +def lower_window_gate_tests(root, source): + s=(source/'g17/freelist.rs').read_text() + start=s.index('static LOWER_FREED');end=s.index('pub(crate) struct Pages {') + gate=s[start:end] + fixture=root/'lower-window-gate-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, Ordering}; +type Result = std::result::Result; +const ENOSPC:i32=-28;const ENOMEM:i32=-12; +static CLOCK: AtomicU64 = AtomicU64::new(1_000_000_000); +fn now_ns()->u64 {CLOCK.load(Ordering::Relaxed)} +mod freelist { +use super::*; +'''+gate+''' +pub(crate) fn freed() {LOWER_FREED.fetch_add(1, Ordering::AcqRel);} +pub(crate) fn place(size: usize, f: impl FnOnce() -> Result) -> Result {place_lower(size, f)} +} +use freelist::*; +fn attempt(size:usize, result:Result, calls:&mut u32)->Result {place(size,||{*calls+=1;result})} +fn main(){ + let mut calls=0; + assert_eq!(attempt(100,Ok(1),&mut calls),Ok(1));assert_eq!(calls,1);assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,2);assert!(lower_pressure()); + for _ in 0..1000 {assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));} + assert_eq!(attempt(200,Ok(1),&mut calls),Err(ENOSPC)); + assert_eq!(calls,2,"a refused size is not placed again while nothing was freed"); + assert_eq!(attempt(50,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"a smaller size still probes"); + assert_eq!(attempt(60,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"the smaller refusal now gates larger sizes"); + freed();assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,4,"freed backing lets a waiter retry"); + // One probe per interval across all callers, then the gate closes again. + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Ok(7),&mut calls),Ok(7));assert_eq!(calls,6);assert!(!lower_pressure(),"a fitting placement clears pressure"); + assert_eq!(attempt(100,Err(ENOMEM),&mut calls),Err(ENOMEM));assert!(!lower_pressure(),"only window refusals gate"); + println!("lower window: refused free-list placements are not repeated until backing is freed or one probe per 100 ms"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'lower-window-gate-controls')],check=True) + subprocess.run([str(root/'lower-window-gate-controls')],check=True) + +def watchdog_publication_guard_tests(source): + # A packet still in publication (capacity or graph waits) must never reach + # pair lookups, the timeout storm count or a collector allocation from its + # watchdog: each of them can stop the runtime for every client. + text = (source / 'g17/runtime/backend.rs').read_text() + at = text.index(' fn timed_out(') + body = text[at:text.index('\n }\n', at)] + guard = body.index('if !packet.is_published() {\n return sched::Status::NoHang;') + assert guard < body.index('self.release_backlog(&packet);') < body.index('self.shared.watchdog_timeout()'), 'timed_out' + for path, name in [('g17/runtime/render/settle.rs', 'pub(in crate::g17) fn timeout_render('), + ('g17/runtime/compute_control.rs', 'pub(super) fn timeout_compute(')]: + text = (source / path).read_text() + at = text.index(name) + body = text[at:text.index('\n }\n', at)] + assert 'let Ok(mut deferred) = DeferredBatch::worker() else {' in body, name + assert 'DeferredBatch::worker()?' not in body, name + print('watchdog: unpublished packets and collector allocation failure never stop the runtime') + def shared_alias_tests(root, source): s=(source/'g17/object.rs').read_text() at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 @@ -380,4 +441,6 @@ def shared_alias_tests(root, source): with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: admission_tests(Path(temporary), DRIVER) queue_tests(Path(temporary), DRIVER) + lower_window_gate_tests(Path(temporary), DRIVER) + watchdog_publication_guard_tests(DRIVER) shared_alias_tests(Path(temporary), DRIVER)