From 34b438de3df9dbd36914d3f4e1d80b7f7a13db7b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:16:35 +1100 Subject: [PATCH] drm/asahi: g17: fence recycled firmware identities and storage G17 hands a new logical queue the lowest free application context (TTBAT id) and QoS ID, and a new context takes the first matching work-state, descriptor or queue object from the device's object cache. A returned identity or object is therefore taken by the very next queue. The firmware names a context's work by those identities: work-state keys are (context id, state word), and queue records carry the context id and QoS ID. An installed compute queue keeps its record after its last owner exits: the scheduler page and work storage stay retained for it, but the context id goes back to the allocator as soon as the scheduler release is consumed. Short-lived Vulkan devices that create and destroy many queues (mesa#184 v1 reserved all eight at vkCreateDevice) hit this on the J616s (M4 Pro, fleet27): aurora: G17: recovery MMU fault bank 0: address 0x7061df9800 write Unmapped unit Unknown(217) (0xd9) level 0 vm_slot 15, slot_mask 0x0 aurora 28a600000.gpu: RTKit: co-processor has crashed (EL1t, ESR 0x96000044, FAR 0x0, x0 = work-state object + 0xf40) The first is a compute-side write through context 15 into the kernel lower arena with no tracked job; the second is the firmware appending to a work list through a node whose links read zero, the state of a freshly recycled work-state object. Give each identity the lifetime of the records that name it: - A context holds its execution identity (mmu::ContextHold), and so does every retained queue record that names it. While a hold remains the identity is handed out only if no other context is free. - Once the firmware has seen an identity, the last hold fences it: it is reused only after the firmware consumed a device-control record published after that point. The firmware consumes control records in order, so that record proves it finished the release that retired the owner. The control producer counts published records across firmware restarts, and every publication and service pass records how far the consumer got. QoS IDs follow the context the same way. - Returned identities are reused oldest first, as the VM slot allocator already does for G13 to G16; mmu::pick_returned is the shared helper. - The object cache reuses the object returned longest ago once its fence is met. A younger one is reused only when its family has no room for a fresh cached mapping, since an uncached one is unmapped as soon as its owner drops it. Under churn every closing queue publishes its own release, so fences clear within a pass or two; in a lull the reuse order keeps fenced identities at the back. Nothing is refused for a fence: when every free context waits, the one with the oldest fence is taken and counted. debugfs capacity reports fenced and overrun counts. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/gpu/drm/asahi/g17/capacity.rs | 2 + drivers/gpu/drm/asahi/g17/channel.rs | 39 +++- drivers/gpu/drm/asahi/g17/context.rs | 87 ++++++-- drivers/gpu/drm/asahi/g17/object.rs | 66 ++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 3 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 1 + drivers/gpu/drm/asahi/mmu.rs | 193 +++++++++++++++++- tools/asahi/g17-growth-tests.py | 3 +- 8 files changed, 359 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 57406e5487a37..405bb65a8eef6 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -69,6 +69,8 @@ impl kernel::debugfs::Writer for View { // locks. Never take a VM/TTBAT lock with the device mutex held here. // The retained UAT owns valid allocator metadata even during shutdown. writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let (fenced, overruns) = uat.execution_contexts_fenced(super::channel::control_witnessed()); + writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?; let mut upper = uat.geometry().kernel_range(); upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; for (name, vm, range) in [ diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index bd9a946d49176..10f69aa32fbb1 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -8,13 +8,48 @@ //! propagate to the device owner. Device-control publication reserves one //! empty slot and orders the record, power tally and producer together. -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; use kernel::{device::Device, prelude::*}; use super::fw::{channels::*, initdata::status}; use super::initdata::InitData; use super::Role; +/// Device-control records published since the driver loaded. Counted across firmware +/// restarts, so it only grows. +static CONTROL_PUBLISHED: AtomicU64 = AtomicU64::new(0); +/// The largest count of published records the firmware is known to have consumed. +static CONTROL_CONSUMED: AtomicU64 = AtomicU64::new(0); + +/// Firmware progress fences for host identities and storage that the firmware may still +/// name after their owner let go of them. +/// +/// The firmware consumes device-control records in order. Once it has consumed a record +/// published after an owner released something, it has finished with every record before +/// that one, including the release that retired the owner. [`fence`](control_fence) is that +/// next record; [`witnessed`](control_witnessed) says how far the firmware got. A fence of +/// zero is always met. +pub(crate) fn control_fence() -> u64 { + CONTROL_PUBLISHED.load(Ordering::Acquire) + 1 +} + +/// The control-ring progress the firmware has shown so far (see [`control_fence`]). +pub(crate) fn control_witnessed() -> u64 { + CONTROL_CONSUMED.load(Ordering::Acquire) +} + +/// Records how far the firmware consumed the control ring. Call under the device mutex, +/// which also serializes publication, with cursors read after any publication by the caller. +/// A restarted firmware has nothing pending, so everything published so far counts. +pub(crate) fn observe_control(consumer: u32, producer: u32) { + if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { + return; + } + let pending = u64::from(producer.wrapping_sub(consumer) & (DEVICE_CONTROL_SLOTS - 1)); + let consumed = CONTROL_PUBLISHED.load(Ordering::Acquire).saturating_sub(pending); + CONTROL_CONSUMED.fetch_max(consumed, Ordering::AcqRel); +} + /// Firmware rings whose contents do not affect host scheduling. #[derive(Copy, Clone, Debug)] enum ReportRing { @@ -282,6 +317,8 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + CONTROL_PUBLISHED.fetch_add(1, Ordering::AcqRel); + observe_control(consumer, next); super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 8a6d9b572dec0..82f1371bc4e17 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -23,39 +23,81 @@ use kernel::{ sync::{Arc, Mutex}, }; -/// QoS IDs belong to one device; the two lowest values are reserved. +const QOS_IDS: usize = 128; + +struct QosState { + used: u128, + /// When each ID was last returned; reuse takes the one returned longest ago. + returns: [u64; QOS_IDS], + /// Firmware progress a returned ID waits for (see `channel::control_fence`). + fences: [u64; QOS_IDS], + clock: u64, +} + +/// QoS IDs belong to one device; the two lowest values are reserved. Queue records name a +/// published context by its QoS ID, so an ID follows the context's execution identity: +/// held while any record names it, then fenced and reused oldest first. #[pin_data] pub(crate) struct QosIds { #[pin] - used: Mutex, + state: Mutex, } impl QosIds { pub(crate) fn new() -> Result> { Arc::pin_init( - pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + pin_init!(Self { + state <- new_mutex!( + QosState { used: 3, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, + "G17 QoS IDs" + ) + }), GFP_KERNEL, ) } - fn acquire(self: &Arc) -> Result { - let mut used = self.used.lock(); - let id = used.trailing_ones(); - if id == 128 { - return Err(super::capacity::refused(super::capacity::Site::QosIds)); - } - *used |= 1u128 << id; - Ok(QosLease { - ids: self.clone(), - id: id as u8, - }) + fn acquire(self: &Arc) -> Result> { + let witnessed = super::channel::control_witnessed(); + let id = { + let mut state = self.state.lock(); + let used = state.used; + let free = (0..QOS_IDS).filter(|&id| used & (1u128 << id) == 0); + let Some((id, _)) = mmu::pick_returned(free, &state.returns, &state.fences, witnessed) + else { + return Err(super::capacity::refused(super::capacity::Site::QosIds)); + }; + state.used |= 1u128 << id; + id + }; + // Drop returns the ID if the allocation fails. + Ok(Arc::new( + QosLease { + ids: self.clone(), + id: id as u8, + seen: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) } } +/// Held by the context and by every retained queue record that names it. struct QosLease { ids: Arc, id: u8, + /// The firmware may have seen this ID. + seen: AtomicBool, } impl Drop for QosLease { fn drop(&mut self) { - *self.ids.used.lock() &= !(1u128 << self.id); + let fence = if self.seen.load(Ordering::Acquire) { + super::channel::control_fence() + } else { + 0 + }; + let mut state = self.ids.state.lock(); + let id = usize::from(self.id); + state.clock += 1; + state.returns[id] = state.clock; + state.fences[id] = fence; + state.used &= !(1u128 << id); } } @@ -120,10 +162,15 @@ impl WorkStorage { pub(crate) struct FirmwarePages { _scheduler: Arc, _work: Arc>, + /// The queue record still names the context's execution identity and QoS ID. + _hold: Arc, + _qos: Arc, } pub(crate) struct Context { execution: Arc, + /// This context's own hold on its execution identity, cloned by retained queue records. + hold: Arc, status: Arc, scheduler: Arc, scheduler_generation: u32, @@ -136,7 +183,7 @@ pub(crate) struct Context { compute_binding_hint: AtomicBool, /// Mixed-engine contexts keep the original inline publication path. render_seen: AtomicBool, - qos: QosLease, + qos: Arc, owner_pid: u32, states: WorkStates, /// Signalled when a work state returns; publishers that found none wait on it. @@ -172,6 +219,7 @@ impl Context { compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); + let hold = execution.hold(super::channel::control_fence)?; let qos = qos.acquire()?; let mut scheduler = alloc.kernel( UAT_PGSZ, @@ -222,6 +270,7 @@ impl Context { Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, + hold, status, scheduler, scheduler_generation, @@ -310,6 +359,8 @@ impl Context { FirmwarePages { _scheduler: self.scheduler.clone(), _work: self.work.clone(), + _hold: self.hold.clone(), + _qos: self.qos.clone(), } } /// Called once the firmware consumed this context's scheduler release. If its work list @@ -362,6 +413,8 @@ impl Context { self.published.load(Ordering::Acquire) } pub(crate) fn mark_published(&self) { + self.hold.mark_seen(); + self.qos.seen.store(true, Ordering::Release); self.published.store(true, Ordering::Release); } @@ -554,7 +607,7 @@ impl Context { impl QosIds { pub(crate) fn used(&self) -> u32 { - (*self.used.lock() & !3u128).count_ones() + (self.state.lock().used & !3u128).count_ones() } } diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 87fdb0e8a62ad..aa30da6fcb41e 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -459,6 +459,11 @@ struct Retained { prefix: usize, cpu: CpuMap, poisoned: bool, + /// Return order: reuse takes the object returned longest ago. + order: u64, + /// Firmware progress after which the previous owner's references are gone + /// (see [`super::channel::control_fence`]). + fence: u64, } struct PoolInner { @@ -472,6 +477,10 @@ struct PoolInner { kept_bytes: usize, /// Kept objects still owned by their users, whose list entries are reserved. kept_reserved: usize, + /// Objects returned so far, ordering the reuse cache. + returns: u64, + /// Reuses that had to take an object whose fence the firmware had not met yet. + fence_overruns: u64, } /// Retained objects past this many are still retained, with one warning. @@ -554,7 +563,7 @@ impl Pool { pin_init!(Self { inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), - kept_bytes: 0, kept_reserved: 0, + kept_bytes: 0, kept_reserved: 0, returns: 0, fence_overruns: 0, }, "g17::object::Pool"), }), GFP_KERNEL, @@ -563,6 +572,12 @@ impl Pool { /// Allocates or reuses a matching object. Capacity pressure leaves a fresh /// mapping uncached, preserving allocation and submission concurrency. + /// + /// The firmware may still follow a previous owner's pointers into a returned object + /// until it has processed that owner's release. Reuse takes the matching object + /// returned longest ago once the firmware consumed a control record published after + /// its return; a younger one is reused only when the family has no room for a fresh + /// cached object, because an uncached one is unmapped as soon as its owner drops it. pub(crate) fn allocate( self: &kernel::sync::Arc, alloc: &Allocator<'_>, @@ -574,22 +589,40 @@ impl Pool { ) -> Result { let family_index = family as usize; let begin = family_index * POOL_SLOTS; + let witnessed = super::channel::control_witnessed(); let reused = { let mut inner = self.inner.lock(); if inner.stopped { return Err(ENODEV); } - let slot = inner.returned[begin..begin + POOL_SLOTS] - .iter_mut() - .find(|entry| { - entry.as_ref().is_some_and(|entry| { - !entry.poisoned - && entry.object.size() == size - && entry.prefix == prefix - && entry.cpu == cpu - }) - }); - slot.and_then(Option::take) + let room = inner.reserved[family_index] < POOL_SLOTS + && inner.bytes.checked_add(size).is_some_and(|bytes| bytes <= POOL_BYTES); + let mut oldest: Option<(usize, u64, bool)> = None; + for (index, entry) in inner.returned[begin..begin + POOL_SLOTS].iter().enumerate() { + let Some(entry) = entry.as_ref().filter(|entry| { + !entry.poisoned + && entry.object.size() == size + && entry.prefix == prefix + && entry.cpu == cpu + }) else { + continue; + }; + let ready = entry.fence <= witnessed; + if oldest.is_none_or(|(_, order, best_ready)| { + (ready && !best_ready) || (ready == best_ready && entry.order < order) + }) { + oldest = Some((begin + index, entry.order, ready)); + } + } + match oldest { + Some((index, _, ready)) if ready || !room => { + if !ready { + inner.fence_overruns += 1; + } + inner.returned[index].take() + } + _ => None, + } }; let lease = || CacheLease { pool: self.clone(), @@ -671,6 +704,9 @@ impl Pool { return; } let begin = family * POOL_SLOTS; + inner.returns += 1; + let order = inner.returns; + let fence = super::channel::control_fence(); for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { if slot.is_none() { *slot = Some(Retained { @@ -678,6 +714,8 @@ impl Pool { prefix: lease.prefix, cpu: lease.cpu, poisoned: lease.poisoned, + order, + fence, }); return; } @@ -776,6 +814,10 @@ impl Pool { entries.iter().flatten().filter(|entry| entry.poisoned).count())?; } writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; + let witnessed = super::channel::control_witnessed(); + writeln!(f, "cached_objects_fenced waiting={} overruns={}", + inner.returned.iter().flatten().filter(|entry| entry.fence > witnessed).count(), + inner.fence_overruns)?; writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0364d5d5055f7..9a96e1adbe654 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -489,6 +489,7 @@ impl Registry { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); + super::channel::observe_control(consumer, producer); let mut freed = 0; for mut queue in self .compute @@ -722,7 +723,7 @@ impl Backend { // Creators hold the device mutex, so neither check above can // have changed; the slot only fails on a broken invariant. let slot = admission.reserve_queue()?; - firmware.uat.new_execution_context(vm).map(|execution| (execution, slot)) + firmware.uat.new_execution_context(vm, super::channel::control_witnessed()).map(|execution| (execution, slot)) } else { Err(ENOSPC) }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 0222ae4c478c8..88252103b676b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -202,6 +202,7 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + crate::g17::channel::observe_control(consumer, producer); let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ad44eba42fb32..4cc9bf4223faf 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1147,6 +1147,22 @@ impl ExecutionContext { self.root.release(); } + /// Takes a hold on this context's identity for a firmware-visible record that names + /// it. `fence` reports the firmware progress to wait for once the last hold drops. + pub(crate) fn hold(&self, fence: fn() -> u64) -> Result> { + let hold = Arc::new( + ContextHold { + inner: self.root.inner.clone(), + id: self.root.id, + seen: AtomicBool::new(false), + fence, + }, + GFP_KERNEL, + )?; + self.root.inner.lock().context_holds[usize::from(self.root.id)] += 1; + Ok(hold) + } + pub(crate) fn is_current(&self) -> bool { let inner = self.root.inner.lock(); if self.root.released.load(Ordering::Acquire) { @@ -1183,7 +1199,10 @@ impl ExecutionRoot { mem::sync(); mem::tlbi_asid(self.id); mem::sync(); - self.inner.lock().execution_contexts &= !(1u64 << id); + let mut inner = self.inner.lock(); + inner.context_clock += 1; + inner.context_returns[id] = inner.context_clock; + inner.execution_contexts &= !(1u64 << id); } else { pr_err!( "MMU: application context {} changed owners while leased\n", @@ -1199,6 +1218,39 @@ impl Drop for ExecutionRoot { } } +/// A firmware-visible record (a queue record, a work list keyed by context) that names an +/// application context. The TTBAT root goes with [`ExecutionContext::release`], but while a +/// hold remains the identity is handed to a new owner only if no other context is free. +/// Once the firmware saw the identity, the last hold fences it: it is reused only after the +/// firmware shows the progress the hold's fence source reported when it dropped. +pub(crate) struct ContextHold { + inner: Arc, + id: u8, + seen: AtomicBool, + fence: fn() -> u64, +} + +impl ContextHold { + /// The firmware may have seen the identity. + pub(crate) fn mark_seen(&self) { + self.seen.store(true, Ordering::Release); + } +} + +impl Drop for ContextHold { + fn drop(&mut self) { + let fence = if self.seen.load(Ordering::Acquire) { (self.fence)() } else { 0 }; + let id = usize::from(self.id); + let mut inner = self.inner.lock(); + inner.context_holds[id] = inner.context_holds[id].saturating_sub(1); + inner.context_fences[id] = inner.context_fences[id].max(fence); + if inner.context_holds[id] == 0 && inner.execution_contexts & (1u64 << id) == 0 { + inner.context_clock += 1; + inner.context_returns[id] = inner.context_clock; + } + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -1795,6 +1847,16 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// When each application context was last returned, on `context_clock`. Allocation + /// takes the free context returned longest ago, as the VM slot allocator does. + context_returns: [u64; UAT_NUM_CTX], + /// Firmware progress each returned context waits for before it may name a new owner. + context_fences: [u64; UAT_NUM_CTX], + /// Firmware-visible records still naming each context (see [`ContextHold`]). + context_holds: [u32; UAT_NUM_CTX], + context_clock: u64, + /// Allocations that had to take a context still held or waiting for its fence. + context_fence_overruns: u64, /// Firmware-observed incarnations of application roots installed by the VM slot allocator. binding_generations: [u64; UAT_NUM_CTX], binding_published: [bool; UAT_NUM_CTX], @@ -1810,6 +1872,17 @@ struct UatShared { } impl UatShared { + /// Context fences with every held context waiting indefinitely. + fn effective_fences(&self) -> [u64; UAT_NUM_CTX] { + let mut fences = self.context_fences; + for (fence, holds) in fences.iter_mut().zip(self.context_holds) { + if holds != 0 { + *fence = u64::MAX; + } + } + fences + } + /// Takes the handoff lock, where this SoC uses it. fn handoff_lock(&self) { if self.quirks.handoff_lock { @@ -3303,6 +3376,11 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + context_returns: [0; UAT_NUM_CTX], + context_fences: [0; UAT_NUM_CTX], + context_holds: [0; UAT_NUM_CTX], + context_clock: 0, + context_fence_overruns: 0, binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], owned_user_roots: [0; UAT_NUM_CTX], @@ -3585,7 +3663,17 @@ impl Uat { /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. - pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { + /// + /// The firmware keys work by context, and installed records may still name a returned + /// context: a context is handed out again only once no [`ContextHold`] remains and the + /// firmware showed progress `witnessed` past its fence, and among those the one returned + /// longest ago is taken. Only when every free context still waits is the one with the + /// oldest fence taken (counted as an overrun) rather than failing a live client. + pub(crate) fn new_execution_context( + &self, + vm: &Vm, + witnessed: u64, + ) -> Result> { // NoPeerRoots: T8152 publishes `t8152::G18ExecutionContext` leases (private empty upper // root) instead of kernel-root upper halves. if self.inner.firmware == UatFirmware::Handoff || self.inner.firmware.is_no_peer() { @@ -3605,7 +3693,16 @@ impl Uat { if free == 0 { return Err(ENOSPC); } - let id = free.trailing_zeros() as usize; + let fences = inner.effective_fences(); + let candidates = (0..UAT_NUM_CTX).filter(|&id| free & (1u64 << id) != 0); + let id = match pick_returned(candidates, &inner.context_returns, &fences, witnessed) { + Some((id, true)) => id, + Some((id, false)) => { + inner.context_fence_overruns += 1; + id + } + None => return Err(ENOSPC), + }; inner.execution_contexts |= 1u64 << id; inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); (id, inner.context_generations[id]) @@ -3827,6 +3924,11 @@ impl Uat { !EXECUTION_CONTEXT_MASK }, context_generations: [0; UAT_NUM_CTX], + context_returns: [0; UAT_NUM_CTX], + context_fences: [0; UAT_NUM_CTX], + context_holds: [0; UAT_NUM_CTX], + context_clock: 0, + context_fence_overruns: 0, binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], owned_user_roots: [0; UAT_NUM_CTX], @@ -4021,4 +4123,89 @@ impl Uat { pub(crate) fn execution_contexts_used(&self) -> u32 { (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() } + + /// Free application contexts still held or waiting for firmware progress `witnessed`, + /// and the allocations that had to take such a context. + pub(crate) fn execution_contexts_fenced(&self, witnessed: u64) -> (u32, u64) { + let inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + let fences = inner.effective_fences(); + let waiting = (0..UAT_NUM_CTX) + .filter(|&id| free & (1u64 << id) != 0 && fences[id] > witnessed) + .count() as u32; + (waiting, inner.context_fence_overruns) + } +} + +/// Picks a returned identity for a new owner, shared by the finite firmware identity +/// spaces (application contexts here, the G17 QoS IDs): among `free`, the one returned +/// longest ago whose fence `witnessed` meets; if none does, the one with the oldest fence. +/// The flag says whether the fence was met. The slot allocator orders VM slots the same way. +pub(crate) fn pick_returned( + free: impl Iterator, + returns: &[u64], + fences: &[u64], + witnessed: u64, +) -> Option<(usize, bool)> { + let mut best: Option = None; + let mut waiting: Option = None; + for id in free { + if fences[id] <= witnessed { + if best.is_none_or(|best| returns[id] < returns[best]) { + best = Some(id); + } + } else if waiting.is_none_or(|waiting| { + (fences[id], returns[id]) < (fences[waiting], returns[waiting]) + }) { + waiting = Some(id); + } + } + best.map(|id| (id, true)).or(waiting.map(|id| (id, false))) +} + +#[kernel::macros::kunit_tests(asahi_mmu_execution_context)] +mod execution_context_tests { + use super::*; + + const FREE: u64 = EXECUTION_CONTEXT_MASK; + + fn pick(free: u64, returns: &[u64], fences: &[u64], witnessed: u64) -> (usize, bool) { + let candidates = (0..UAT_NUM_CTX).filter(|&id| free & (1u64 << id) != 0); + pick_returned(candidates, returns, fences, witnessed).unwrap() + } + + #[test] + fn a_returned_context_goes_to_the_back_of_the_line() { + let mut returns = [0; UAT_NUM_CTX]; + let fences = [0; UAT_NUM_CTX]; + // Never-used contexts (return time 0) go first, lowest id among equals. + assert_eq!(pick(FREE, &returns, &fences, 0), (5, true)); + // Context 5 was just returned: the next never-used one is taken instead. + returns[5] = 1; + assert_eq!(pick(FREE, &returns, &fences, 0), (6, true)); + // Every context returned once: the one returned longest ago wins. + for (id, time) in returns.iter_mut().enumerate() { + *time = 100 - id as u64; + } + assert_eq!(pick(FREE, &returns, &fences, 0), (62, true)); + assert_eq!(pick(FREE & !(1 << 62), &returns, &fences, 0), (61, true)); + } + + #[test] + fn a_fenced_context_waits_for_firmware_progress() { + let returns = [0; UAT_NUM_CTX]; + let mut fences = [0; UAT_NUM_CTX]; + let only = (1u64 << 7) | (1u64 << 9); + fences[7] = 10; + assert_eq!(pick(only, &returns, &fences, 9), (9, true)); + assert_eq!(pick(1 << 7, &returns, &fences, 10), (7, true)); + // Every free context waits: the oldest fence is taken and reported. + fences[9] = 12; + assert_eq!(pick(only, &returns, &fences, 9), (7, false)); + assert_eq!(pick(only, &returns, &fences, 11), (7, true)); + // A held context (fence u64::MAX) goes last. + fences[7] = u64::MAX; + assert_eq!(pick(only, &returns, &fences, 11), (9, false)); + assert!(pick_returned(core::iter::empty(), &returns, &fences, 0).is_none()); + } } diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index 2c46ccbf6359a..2150f121eaa6f 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -48,7 +48,7 @@ def fixture(): ''' + item(source,'pub(crate) fn release_owner(') + item(source,'pub(crate) fn reusable(') + '\n}\n' methods = '\n'.join(item(source, sig) for sig in ['pub(crate) fn begin_growth(', 'pub(crate) fn finish_growth(', 'pub(crate) fn growth_token(', 'pub(crate) fn abort_growth(', 'pub(crate) fn observe_vacancy(']) replace = item((ROOT / 'freelist.rs').read_text(), 'pub(crate) fn replace_pages(') - observe = item((ROOT / 'runtime.rs').read_text(), 'pub(super) fn observe_pool_releases(').replace('pub(super) ', '').replace('super::initdata::', 'initdata::') + observe = item((ROOT / 'runtime.rs').read_text(), 'pub(super) fn observe_pool_releases(').replace('pub(super) ', '').replace('super::initdata::', 'initdata::').replace('super::channel::', 'g17::channel::') return text + r''' const EAGAIN:i32=-11;const EFAULT:i32=-14; static CONSUMER:AtomicU32=AtomicU32::new(0); @@ -58,6 +58,7 @@ def fixture(): mod mmu {pub type KernelMapping=u64;} mod g17 { pub fn now_ns()->u64{crate::now_ns()} + pub mod channel {pub fn observe_control(_:u32,_:u32){}} pub mod freelist {pub(crate) use crate::Pages; pub fn control_consumed(c:u32,_:u32,t:u32)->crate::Result{Ok(c==t)}} pub mod initdata { pub struct InitData {pub fail:bool}