diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 57406e5487a37..405bb65a8eef6 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -69,6 +69,8 @@ impl kernel::debugfs::Writer for View { // locks. Never take a VM/TTBAT lock with the device mutex held here. // The retained UAT owns valid allocator metadata even during shutdown. writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let (fenced, overruns) = uat.execution_contexts_fenced(super::channel::control_witnessed()); + writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?; let mut upper = uat.geometry().kernel_range(); upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; for (name, vm, range) in [ diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index bd9a946d49176..10f69aa32fbb1 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -8,13 +8,48 @@ //! propagate to the device owner. Device-control publication reserves one //! empty slot and orders the record, power tally and producer together. -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; use kernel::{device::Device, prelude::*}; use super::fw::{channels::*, initdata::status}; use super::initdata::InitData; use super::Role; +/// Device-control records published since the driver loaded. Counted across firmware +/// restarts, so it only grows. +static CONTROL_PUBLISHED: AtomicU64 = AtomicU64::new(0); +/// The largest count of published records the firmware is known to have consumed. +static CONTROL_CONSUMED: AtomicU64 = AtomicU64::new(0); + +/// Firmware progress fences for host identities and storage that the firmware may still +/// name after their owner let go of them. +/// +/// The firmware consumes device-control records in order. Once it has consumed a record +/// published after an owner released something, it has finished with every record before +/// that one, including the release that retired the owner. [`fence`](control_fence) is that +/// next record; [`witnessed`](control_witnessed) says how far the firmware got. A fence of +/// zero is always met. +pub(crate) fn control_fence() -> u64 { + CONTROL_PUBLISHED.load(Ordering::Acquire) + 1 +} + +/// The control-ring progress the firmware has shown so far (see [`control_fence`]). +pub(crate) fn control_witnessed() -> u64 { + CONTROL_CONSUMED.load(Ordering::Acquire) +} + +/// Records how far the firmware consumed the control ring. Call under the device mutex, +/// which also serializes publication, with cursors read after any publication by the caller. +/// A restarted firmware has nothing pending, so everything published so far counts. +pub(crate) fn observe_control(consumer: u32, producer: u32) { + if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { + return; + } + let pending = u64::from(producer.wrapping_sub(consumer) & (DEVICE_CONTROL_SLOTS - 1)); + let consumed = CONTROL_PUBLISHED.load(Ordering::Acquire).saturating_sub(pending); + CONTROL_CONSUMED.fetch_max(consumed, Ordering::AcqRel); +} + /// Firmware rings whose contents do not affect host scheduling. #[derive(Copy, Clone, Debug)] enum ReportRing { @@ -282,6 +317,8 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + CONTROL_PUBLISHED.fetch_add(1, Ordering::AcqRel); + observe_control(consumer, next); super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 8a6d9b572dec0..82f1371bc4e17 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -23,39 +23,81 @@ use kernel::{ sync::{Arc, Mutex}, }; -/// QoS IDs belong to one device; the two lowest values are reserved. +const QOS_IDS: usize = 128; + +struct QosState { + used: u128, + /// When each ID was last returned; reuse takes the one returned longest ago. + returns: [u64; QOS_IDS], + /// Firmware progress a returned ID waits for (see `channel::control_fence`). + fences: [u64; QOS_IDS], + clock: u64, +} + +/// QoS IDs belong to one device; the two lowest values are reserved. Queue records name a +/// published context by its QoS ID, so an ID follows the context's execution identity: +/// held while any record names it, then fenced and reused oldest first. #[pin_data] pub(crate) struct QosIds { #[pin] - used: Mutex, + state: Mutex, } impl QosIds { pub(crate) fn new() -> Result> { Arc::pin_init( - pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + pin_init!(Self { + state <- new_mutex!( + QosState { used: 3, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 }, + "G17 QoS IDs" + ) + }), GFP_KERNEL, ) } - fn acquire(self: &Arc) -> Result { - let mut used = self.used.lock(); - let id = used.trailing_ones(); - if id == 128 { - return Err(super::capacity::refused(super::capacity::Site::QosIds)); - } - *used |= 1u128 << id; - Ok(QosLease { - ids: self.clone(), - id: id as u8, - }) + fn acquire(self: &Arc) -> Result> { + let witnessed = super::channel::control_witnessed(); + let id = { + let mut state = self.state.lock(); + let used = state.used; + let free = (0..QOS_IDS).filter(|&id| used & (1u128 << id) == 0); + let Some((id, _)) = mmu::pick_returned(free, &state.returns, &state.fences, witnessed) + else { + return Err(super::capacity::refused(super::capacity::Site::QosIds)); + }; + state.used |= 1u128 << id; + id + }; + // Drop returns the ID if the allocation fails. + Ok(Arc::new( + QosLease { + ids: self.clone(), + id: id as u8, + seen: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) } } +/// Held by the context and by every retained queue record that names it. struct QosLease { ids: Arc, id: u8, + /// The firmware may have seen this ID. + seen: AtomicBool, } impl Drop for QosLease { fn drop(&mut self) { - *self.ids.used.lock() &= !(1u128 << self.id); + let fence = if self.seen.load(Ordering::Acquire) { + super::channel::control_fence() + } else { + 0 + }; + let mut state = self.ids.state.lock(); + let id = usize::from(self.id); + state.clock += 1; + state.returns[id] = state.clock; + state.fences[id] = fence; + state.used &= !(1u128 << id); } } @@ -120,10 +162,15 @@ impl WorkStorage { pub(crate) struct FirmwarePages { _scheduler: Arc, _work: Arc>, + /// The queue record still names the context's execution identity and QoS ID. + _hold: Arc, + _qos: Arc, } pub(crate) struct Context { execution: Arc, + /// This context's own hold on its execution identity, cloned by retained queue records. + hold: Arc, status: Arc, scheduler: Arc, scheduler_generation: u32, @@ -136,7 +183,7 @@ pub(crate) struct Context { compute_binding_hint: AtomicBool, /// Mixed-engine contexts keep the original inline publication path. render_seen: AtomicBool, - qos: QosLease, + qos: Arc, owner_pid: u32, states: WorkStates, /// Signalled when a work state returns; publishers that found none wait on it. @@ -172,6 +219,7 @@ impl Context { compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); + let hold = execution.hold(super::channel::control_fence)?; let qos = qos.acquire()?; let mut scheduler = alloc.kernel( UAT_PGSZ, @@ -222,6 +270,7 @@ impl Context { Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, + hold, status, scheduler, scheduler_generation, @@ -310,6 +359,8 @@ impl Context { FirmwarePages { _scheduler: self.scheduler.clone(), _work: self.work.clone(), + _hold: self.hold.clone(), + _qos: self.qos.clone(), } } /// Called once the firmware consumed this context's scheduler release. If its work list @@ -362,6 +413,8 @@ impl Context { self.published.load(Ordering::Acquire) } pub(crate) fn mark_published(&self) { + self.hold.mark_seen(); + self.qos.seen.store(true, Ordering::Release); self.published.store(true, Ordering::Release); } @@ -554,7 +607,7 @@ impl Context { impl QosIds { pub(crate) fn used(&self) -> u32 { - (*self.used.lock() & !3u128).count_ones() + (self.state.lock().used & !3u128).count_ones() } } diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 87fdb0e8a62ad..aa30da6fcb41e 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -459,6 +459,11 @@ struct Retained { prefix: usize, cpu: CpuMap, poisoned: bool, + /// Return order: reuse takes the object returned longest ago. + order: u64, + /// Firmware progress after which the previous owner's references are gone + /// (see [`super::channel::control_fence`]). + fence: u64, } struct PoolInner { @@ -472,6 +477,10 @@ struct PoolInner { kept_bytes: usize, /// Kept objects still owned by their users, whose list entries are reserved. kept_reserved: usize, + /// Objects returned so far, ordering the reuse cache. + returns: u64, + /// Reuses that had to take an object whose fence the firmware had not met yet. + fence_overruns: u64, } /// Retained objects past this many are still retained, with one warning. @@ -554,7 +563,7 @@ impl Pool { pin_init!(Self { inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), - kept_bytes: 0, kept_reserved: 0, + kept_bytes: 0, kept_reserved: 0, returns: 0, fence_overruns: 0, }, "g17::object::Pool"), }), GFP_KERNEL, @@ -563,6 +572,12 @@ impl Pool { /// Allocates or reuses a matching object. Capacity pressure leaves a fresh /// mapping uncached, preserving allocation and submission concurrency. + /// + /// The firmware may still follow a previous owner's pointers into a returned object + /// until it has processed that owner's release. Reuse takes the matching object + /// returned longest ago once the firmware consumed a control record published after + /// its return; a younger one is reused only when the family has no room for a fresh + /// cached object, because an uncached one is unmapped as soon as its owner drops it. pub(crate) fn allocate( self: &kernel::sync::Arc, alloc: &Allocator<'_>, @@ -574,22 +589,40 @@ impl Pool { ) -> Result { let family_index = family as usize; let begin = family_index * POOL_SLOTS; + let witnessed = super::channel::control_witnessed(); let reused = { let mut inner = self.inner.lock(); if inner.stopped { return Err(ENODEV); } - let slot = inner.returned[begin..begin + POOL_SLOTS] - .iter_mut() - .find(|entry| { - entry.as_ref().is_some_and(|entry| { - !entry.poisoned - && entry.object.size() == size - && entry.prefix == prefix - && entry.cpu == cpu - }) - }); - slot.and_then(Option::take) + let room = inner.reserved[family_index] < POOL_SLOTS + && inner.bytes.checked_add(size).is_some_and(|bytes| bytes <= POOL_BYTES); + let mut oldest: Option<(usize, u64, bool)> = None; + for (index, entry) in inner.returned[begin..begin + POOL_SLOTS].iter().enumerate() { + let Some(entry) = entry.as_ref().filter(|entry| { + !entry.poisoned + && entry.object.size() == size + && entry.prefix == prefix + && entry.cpu == cpu + }) else { + continue; + }; + let ready = entry.fence <= witnessed; + if oldest.is_none_or(|(_, order, best_ready)| { + (ready && !best_ready) || (ready == best_ready && entry.order < order) + }) { + oldest = Some((begin + index, entry.order, ready)); + } + } + match oldest { + Some((index, _, ready)) if ready || !room => { + if !ready { + inner.fence_overruns += 1; + } + inner.returned[index].take() + } + _ => None, + } }; let lease = || CacheLease { pool: self.clone(), @@ -671,6 +704,9 @@ impl Pool { return; } let begin = family * POOL_SLOTS; + inner.returns += 1; + let order = inner.returns; + let fence = super::channel::control_fence(); for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { if slot.is_none() { *slot = Some(Retained { @@ -678,6 +714,8 @@ impl Pool { prefix: lease.prefix, cpu: lease.cpu, poisoned: lease.poisoned, + order, + fence, }); return; } @@ -776,6 +814,10 @@ impl Pool { entries.iter().flatten().filter(|entry| entry.poisoned).count())?; } writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; + let witnessed = super::channel::control_witnessed(); + writeln!(f, "cached_objects_fenced waiting={} overruns={}", + inner.returned.iter().flatten().filter(|entry| entry.fence > witnessed).count(), + inner.fence_overruns)?; writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0364d5d5055f7..9a96e1adbe654 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -489,6 +489,7 @@ impl Registry { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); + super::channel::observe_control(consumer, producer); let mut freed = 0; for mut queue in self .compute @@ -722,7 +723,7 @@ impl Backend { // Creators hold the device mutex, so neither check above can // have changed; the slot only fails on a broken invariant. let slot = admission.reserve_queue()?; - firmware.uat.new_execution_context(vm).map(|execution| (execution, slot)) + firmware.uat.new_execution_context(vm, super::channel::control_witnessed()).map(|execution| (execution, slot)) } else { Err(ENOSPC) }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 0222ae4c478c8..88252103b676b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -202,6 +202,7 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + crate::g17::channel::observe_control(consumer, producer); let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ad44eba42fb32..4cc9bf4223faf 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1147,6 +1147,22 @@ impl ExecutionContext { self.root.release(); } + /// Takes a hold on this context's identity for a firmware-visible record that names + /// it. `fence` reports the firmware progress to wait for once the last hold drops. + pub(crate) fn hold(&self, fence: fn() -> u64) -> Result> { + let hold = Arc::new( + ContextHold { + inner: self.root.inner.clone(), + id: self.root.id, + seen: AtomicBool::new(false), + fence, + }, + GFP_KERNEL, + )?; + self.root.inner.lock().context_holds[usize::from(self.root.id)] += 1; + Ok(hold) + } + pub(crate) fn is_current(&self) -> bool { let inner = self.root.inner.lock(); if self.root.released.load(Ordering::Acquire) { @@ -1183,7 +1199,10 @@ impl ExecutionRoot { mem::sync(); mem::tlbi_asid(self.id); mem::sync(); - self.inner.lock().execution_contexts &= !(1u64 << id); + let mut inner = self.inner.lock(); + inner.context_clock += 1; + inner.context_returns[id] = inner.context_clock; + inner.execution_contexts &= !(1u64 << id); } else { pr_err!( "MMU: application context {} changed owners while leased\n", @@ -1199,6 +1218,39 @@ impl Drop for ExecutionRoot { } } +/// A firmware-visible record (a queue record, a work list keyed by context) that names an +/// application context. The TTBAT root goes with [`ExecutionContext::release`], but while a +/// hold remains the identity is handed to a new owner only if no other context is free. +/// Once the firmware saw the identity, the last hold fences it: it is reused only after the +/// firmware shows the progress the hold's fence source reported when it dropped. +pub(crate) struct ContextHold { + inner: Arc, + id: u8, + seen: AtomicBool, + fence: fn() -> u64, +} + +impl ContextHold { + /// The firmware may have seen the identity. + pub(crate) fn mark_seen(&self) { + self.seen.store(true, Ordering::Release); + } +} + +impl Drop for ContextHold { + fn drop(&mut self) { + let fence = if self.seen.load(Ordering::Acquire) { (self.fence)() } else { 0 }; + let id = usize::from(self.id); + let mut inner = self.inner.lock(); + inner.context_holds[id] = inner.context_holds[id].saturating_sub(1); + inner.context_fences[id] = inner.context_fences[id].max(fence); + if inner.context_holds[id] == 0 && inner.execution_contexts & (1u64 << id) == 0 { + inner.context_clock += 1; + inner.context_returns[id] = inner.context_clock; + } + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -1795,6 +1847,16 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// When each application context was last returned, on `context_clock`. Allocation + /// takes the free context returned longest ago, as the VM slot allocator does. + context_returns: [u64; UAT_NUM_CTX], + /// Firmware progress each returned context waits for before it may name a new owner. + context_fences: [u64; UAT_NUM_CTX], + /// Firmware-visible records still naming each context (see [`ContextHold`]). + context_holds: [u32; UAT_NUM_CTX], + context_clock: u64, + /// Allocations that had to take a context still held or waiting for its fence. + context_fence_overruns: u64, /// Firmware-observed incarnations of application roots installed by the VM slot allocator. binding_generations: [u64; UAT_NUM_CTX], binding_published: [bool; UAT_NUM_CTX], @@ -1810,6 +1872,17 @@ struct UatShared { } impl UatShared { + /// Context fences with every held context waiting indefinitely. + fn effective_fences(&self) -> [u64; UAT_NUM_CTX] { + let mut fences = self.context_fences; + for (fence, holds) in fences.iter_mut().zip(self.context_holds) { + if holds != 0 { + *fence = u64::MAX; + } + } + fences + } + /// Takes the handoff lock, where this SoC uses it. fn handoff_lock(&self) { if self.quirks.handoff_lock { @@ -3303,6 +3376,11 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + context_returns: [0; UAT_NUM_CTX], + context_fences: [0; UAT_NUM_CTX], + context_holds: [0; UAT_NUM_CTX], + context_clock: 0, + context_fence_overruns: 0, binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], owned_user_roots: [0; UAT_NUM_CTX], @@ -3585,7 +3663,17 @@ impl Uat { /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. - pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { + /// + /// The firmware keys work by context, and installed records may still name a returned + /// context: a context is handed out again only once no [`ContextHold`] remains and the + /// firmware showed progress `witnessed` past its fence, and among those the one returned + /// longest ago is taken. Only when every free context still waits is the one with the + /// oldest fence taken (counted as an overrun) rather than failing a live client. + pub(crate) fn new_execution_context( + &self, + vm: &Vm, + witnessed: u64, + ) -> Result> { // NoPeerRoots: T8152 publishes `t8152::G18ExecutionContext` leases (private empty upper // root) instead of kernel-root upper halves. if self.inner.firmware == UatFirmware::Handoff || self.inner.firmware.is_no_peer() { @@ -3605,7 +3693,16 @@ impl Uat { if free == 0 { return Err(ENOSPC); } - let id = free.trailing_zeros() as usize; + let fences = inner.effective_fences(); + let candidates = (0..UAT_NUM_CTX).filter(|&id| free & (1u64 << id) != 0); + let id = match pick_returned(candidates, &inner.context_returns, &fences, witnessed) { + Some((id, true)) => id, + Some((id, false)) => { + inner.context_fence_overruns += 1; + id + } + None => return Err(ENOSPC), + }; inner.execution_contexts |= 1u64 << id; inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); (id, inner.context_generations[id]) @@ -3827,6 +3924,11 @@ impl Uat { !EXECUTION_CONTEXT_MASK }, context_generations: [0; UAT_NUM_CTX], + context_returns: [0; UAT_NUM_CTX], + context_fences: [0; UAT_NUM_CTX], + context_holds: [0; UAT_NUM_CTX], + context_clock: 0, + context_fence_overruns: 0, binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], owned_user_roots: [0; UAT_NUM_CTX], @@ -4021,4 +4123,89 @@ impl Uat { pub(crate) fn execution_contexts_used(&self) -> u32 { (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() } + + /// Free application contexts still held or waiting for firmware progress `witnessed`, + /// and the allocations that had to take such a context. + pub(crate) fn execution_contexts_fenced(&self, witnessed: u64) -> (u32, u64) { + let inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + let fences = inner.effective_fences(); + let waiting = (0..UAT_NUM_CTX) + .filter(|&id| free & (1u64 << id) != 0 && fences[id] > witnessed) + .count() as u32; + (waiting, inner.context_fence_overruns) + } +} + +/// Picks a returned identity for a new owner, shared by the finite firmware identity +/// spaces (application contexts here, the G17 QoS IDs): among `free`, the one returned +/// longest ago whose fence `witnessed` meets; if none does, the one with the oldest fence. +/// The flag says whether the fence was met. The slot allocator orders VM slots the same way. +pub(crate) fn pick_returned( + free: impl Iterator, + returns: &[u64], + fences: &[u64], + witnessed: u64, +) -> Option<(usize, bool)> { + let mut best: Option = None; + let mut waiting: Option = None; + for id in free { + if fences[id] <= witnessed { + if best.is_none_or(|best| returns[id] < returns[best]) { + best = Some(id); + } + } else if waiting.is_none_or(|waiting| { + (fences[id], returns[id]) < (fences[waiting], returns[waiting]) + }) { + waiting = Some(id); + } + } + best.map(|id| (id, true)).or(waiting.map(|id| (id, false))) +} + +#[kernel::macros::kunit_tests(asahi_mmu_execution_context)] +mod execution_context_tests { + use super::*; + + const FREE: u64 = EXECUTION_CONTEXT_MASK; + + fn pick(free: u64, returns: &[u64], fences: &[u64], witnessed: u64) -> (usize, bool) { + let candidates = (0..UAT_NUM_CTX).filter(|&id| free & (1u64 << id) != 0); + pick_returned(candidates, returns, fences, witnessed).unwrap() + } + + #[test] + fn a_returned_context_goes_to_the_back_of_the_line() { + let mut returns = [0; UAT_NUM_CTX]; + let fences = [0; UAT_NUM_CTX]; + // Never-used contexts (return time 0) go first, lowest id among equals. + assert_eq!(pick(FREE, &returns, &fences, 0), (5, true)); + // Context 5 was just returned: the next never-used one is taken instead. + returns[5] = 1; + assert_eq!(pick(FREE, &returns, &fences, 0), (6, true)); + // Every context returned once: the one returned longest ago wins. + for (id, time) in returns.iter_mut().enumerate() { + *time = 100 - id as u64; + } + assert_eq!(pick(FREE, &returns, &fences, 0), (62, true)); + assert_eq!(pick(FREE & !(1 << 62), &returns, &fences, 0), (61, true)); + } + + #[test] + fn a_fenced_context_waits_for_firmware_progress() { + let returns = [0; UAT_NUM_CTX]; + let mut fences = [0; UAT_NUM_CTX]; + let only = (1u64 << 7) | (1u64 << 9); + fences[7] = 10; + assert_eq!(pick(only, &returns, &fences, 9), (9, true)); + assert_eq!(pick(1 << 7, &returns, &fences, 10), (7, true)); + // Every free context waits: the oldest fence is taken and reported. + fences[9] = 12; + assert_eq!(pick(only, &returns, &fences, 9), (7, false)); + assert_eq!(pick(only, &returns, &fences, 11), (7, true)); + // A held context (fence u64::MAX) goes last. + fences[7] = u64::MAX; + assert_eq!(pick(only, &returns, &fences, 11), (9, false)); + assert!(pick_returned(core::iter::empty(), &returns, &fences, 0).is_none()); + } } diff --git a/tools/asahi/g17-growth-tests.py b/tools/asahi/g17-growth-tests.py index 2c46ccbf6359a..2150f121eaa6f 100644 --- a/tools/asahi/g17-growth-tests.py +++ b/tools/asahi/g17-growth-tests.py @@ -48,7 +48,7 @@ def fixture(): ''' + item(source,'pub(crate) fn release_owner(') + item(source,'pub(crate) fn reusable(') + '\n}\n' methods = '\n'.join(item(source, sig) for sig in ['pub(crate) fn begin_growth(', 'pub(crate) fn finish_growth(', 'pub(crate) fn growth_token(', 'pub(crate) fn abort_growth(', 'pub(crate) fn observe_vacancy(']) replace = item((ROOT / 'freelist.rs').read_text(), 'pub(crate) fn replace_pages(') - observe = item((ROOT / 'runtime.rs').read_text(), 'pub(super) fn observe_pool_releases(').replace('pub(super) ', '').replace('super::initdata::', 'initdata::') + observe = item((ROOT / 'runtime.rs').read_text(), 'pub(super) fn observe_pool_releases(').replace('pub(super) ', '').replace('super::initdata::', 'initdata::').replace('super::channel::', 'g17::channel::') return text + r''' const EAGAIN:i32=-11;const EFAULT:i32=-14; static CONSUMER:AtomicU32=AtomicU32::new(0); @@ -58,6 +58,7 @@ def fixture(): mod mmu {pub type KernelMapping=u64;} mod g17 { pub fn now_ns()->u64{crate::now_ns()} + pub mod channel {pub fn observe_control(_:u32,_:u32){}} pub mod freelist {pub(crate) use crate::Pages; pub fn control_consumed(c:u32,_:u32,t:u32)->crate::Result{Ok(c==t)}} pub mod initdata { pub struct InitData {pub fail:bool}