drm/asahi: g17: fence recycled firmware identities and storage

drivers/gpu/drm/asahi/g17/capacity.rs

@@ -69,6 +69,8 @@ impl kernel::debugfs::Writer for View {
         // locks. Never take a VM/TTBAT lock with the device mutex held here.
         // The retained UAT owns valid allocator metadata even during shutdown.
         writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?;
+        let (fenced, overruns) = uat.execution_contexts_fenced(super::channel::control_witnessed());
+        writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?;
         let mut upper = uat.geometry().kernel_range();
         upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET;
         for (name, vm, range) in [

drivers/gpu/drm/asahi/g17/channel.rs

@@ -8,13 +8,48 @@
 //! propagate to the device owner. Device-control publication reserves one
 //! empty slot and orders the record, power tally and producer together.
 
-use core::sync::atomic::{fence, Ordering};
+use core::sync::atomic::{fence, AtomicU64, Ordering};
 use kernel::{device::Device, prelude::*};
 
 use super::fw::{channels::*, initdata::status};
 use super::initdata::InitData;
 use super::Role;
 
+/// Device-control records published since the driver loaded. Counted across firmware
+/// restarts, so it only grows.
+static CONTROL_PUBLISHED: AtomicU64 = AtomicU64::new(0);
+/// The largest count of published records the firmware is known to have consumed.
+static CONTROL_CONSUMED: AtomicU64 = AtomicU64::new(0);
+
+/// Firmware progress fences for host identities and storage that the firmware may still
+/// name after their owner let go of them.
+///
+/// The firmware consumes device-control records in order. Once it has consumed a record
+/// published after an owner released something, it has finished with every record before
+/// that one, including the release that retired the owner. [`fence`](control_fence) is that
+/// next record; [`witnessed`](control_witnessed) says how far the firmware got. A fence of
+/// zero is always met.
+pub(crate) fn control_fence() -> u64 {
+    CONTROL_PUBLISHED.load(Ordering::Acquire) + 1
+}
+
+/// The control-ring progress the firmware has shown so far (see [`control_fence`]).
+pub(crate) fn control_witnessed() -> u64 {
+    CONTROL_CONSUMED.load(Ordering::Acquire)
+}
+
+/// Records how far the firmware consumed the control ring. Call under the device mutex,
+/// which also serializes publication, with cursors read after any publication by the caller.
+/// A restarted firmware has nothing pending, so everything published so far counts.
+pub(crate) fn observe_control(consumer: u32, producer: u32) {
+    if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS {
+        return;
+    }
+    let pending = u64::from(producer.wrapping_sub(consumer) & (DEVICE_CONTROL_SLOTS - 1));
+    let consumed = CONTROL_PUBLISHED.load(Ordering::Acquire).saturating_sub(pending);
+    CONTROL_CONSUMED.fetch_max(consumed, Ordering::AcqRel);
+}
+
 /// Firmware rings whose contents do not affect host scheduling.
 #[derive(Copy, Clone, Debug)]
 enum ReportRing {
@@ -282,6 +317,8 @@ impl Rings {
             record.opcode().asserts_gpu_power(),
             next,
         )?;
+        CONTROL_PUBLISHED.fetch_add(1, Ordering::AcqRel);
+        observe_control(consumer, next);
         super::trace::message(super::trace::CONTROL, record.opcode() as u64);
         Ok(next)
     }

drivers/gpu/drm/asahi/g17/context.rs

@@ -23,39 +23,81 @@ use kernel::{
     sync::{Arc, Mutex},
 };
 
-/// QoS IDs belong to one device; the two lowest values are reserved.
+const QOS_IDS: usize = 128;
+
+struct QosState {
+    used: u128,
+    /// When each ID was last returned; reuse takes the one returned longest ago.
+    returns: [u64; QOS_IDS],
+    /// Firmware progress a returned ID waits for (see `channel::control_fence`).
+    fences: [u64; QOS_IDS],
+    clock: u64,
+}
+
+/// QoS IDs belong to one device; the two lowest values are reserved. Queue records name a
+/// published context by its QoS ID, so an ID follows the context's execution identity:
+/// held while any record names it, then fenced and reused oldest first.
 #[pin_data]
 pub(crate) struct QosIds {
     #[pin]
-    used: Mutex<u128>,
+    state: Mutex<QosState>,
 }
 impl QosIds {
     pub(crate) fn new() -> Result<Arc<Self>> {
         Arc::pin_init(
-            pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }),
+            pin_init!(Self {
+                state <- new_mutex!(
+                    QosState { used: 3, returns: [0; QOS_IDS], fences: [0; QOS_IDS], clock: 0 },
+                    "G17 QoS IDs"
+                )
+            }),
             GFP_KERNEL,
         )
     }
-    fn acquire(self: &Arc<Self>) -> Result<QosLease> {
-        let mut used = self.used.lock();
-        let id = used.trailing_ones();
-        if id == 128 {
-            return Err(super::capacity::refused(super::capacity::Site::QosIds));
-        }
-        *used |= 1u128 << id;
-        Ok(QosLease {
-            ids: self.clone(),
-            id: id as u8,
-        })
+    fn acquire(self: &Arc<Self>) -> Result<Arc<QosLease>> {
+        let witnessed = super::channel::control_witnessed();
+        let id = {
+            let mut state = self.state.lock();
+            let used = state.used;
+            let free = (0..QOS_IDS).filter(|&id| used & (1u128 << id) == 0);
+            let Some((id, _)) = mmu::pick_returned(free, &state.returns, &state.fences, witnessed)
+            else {
+                return Err(super::capacity::refused(super::capacity::Site::QosIds));
+            };
+            state.used |= 1u128 << id;
+            id
+        };
+        // Drop returns the ID if the allocation fails.
+        Ok(Arc::new(
+            QosLease {
+                ids: self.clone(),
+                id: id as u8,
+                seen: AtomicBool::new(false),
+            },
+            GFP_KERNEL,
+        )?)
     }
 }
+/// Held by the context and by every retained queue record that names it.
 struct QosLease {
     ids: Arc<QosIds>,
     id: u8,
+    /// The firmware may have seen this ID.
+    seen: AtomicBool,
 }
 impl Drop for QosLease {
     fn drop(&mut self) {
-        *self.ids.used.lock() &= !(1u128 << self.id);
+        let fence = if self.seen.load(Ordering::Acquire) {
+            super::channel::control_fence()
+        } else {
+            0
+        };
+        let mut state = self.ids.state.lock();
+        let id = usize::from(self.id);
+        state.clock += 1;
+        state.returns[id] = state.clock;
+        state.fences[id] = fence;
+        state.used &= !(1u128 << id);
     }
 }
 
@@ -120,10 +162,15 @@ impl WorkStorage {
 pub(crate) struct FirmwarePages {
     _scheduler: Arc<KernelObject>,
     _work: Arc<Mutex<WorkStorage>>,
+    /// The queue record still names the context's execution identity and QoS ID.
+    _hold: Arc<mmu::ContextHold>,
+    _qos: Arc<QosLease>,
 }
 
 pub(crate) struct Context {
     execution: Arc<ExecutionContext>,
+    /// This context's own hold on its execution identity, cloned by retained queue records.
+    hold: Arc<mmu::ContextHold>,
     status: Arc<VmStatus>,
     scheduler: Arc<KernelObject>,
     scheduler_generation: u32,
@@ -136,7 +183,7 @@ pub(crate) struct Context {
     compute_binding_hint: AtomicBool,
     /// Mixed-engine contexts keep the original inline publication path.
     render_seen: AtomicBool,
-    qos: QosLease,
+    qos: Arc<QosLease>,
     owner_pid: u32,
     states: WorkStates,
     /// Signalled when a work state returns; publishers that found none wait on it.
@@ -172,6 +219,7 @@ impl Context {
         compute_returns: Arc<super::availability::Availability>,
     ) -> Result<Self> {
         let status = execution.vm().status().ok_or(EINVAL)?.clone();
+        let hold = execution.hold(super::channel::control_fence)?;
         let qos = qos.acquire()?;
         let mut scheduler = alloc.kernel(
             UAT_PGSZ,
@@ -222,6 +270,7 @@ impl Context {
             Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?;
         Ok(Self {
             execution,
+            hold,
             status,
             scheduler,
             scheduler_generation,
@@ -310,6 +359,8 @@ impl Context {
         FirmwarePages {
             _scheduler: self.scheduler.clone(),
             _work: self.work.clone(),
+            _hold: self.hold.clone(),
+            _qos: self.qos.clone(),
         }
     }
     /// Called once the firmware consumed this context's scheduler release. If its work list
@@ -362,6 +413,8 @@ impl Context {
         self.published.load(Ordering::Acquire)
     }
     pub(crate) fn mark_published(&self) {
+        self.hold.mark_seen();
+        self.qos.seen.store(true, Ordering::Release);
         self.published.store(true, Ordering::Release);
     }
 
@@ -554,7 +607,7 @@ impl Context {
 
 impl QosIds {
     pub(crate) fn used(&self) -> u32 {
-        (*self.used.lock() & !3u128).count_ones()
+        (self.state.lock().used & !3u128).count_ones()
     }
 }
 

drivers/gpu/drm/asahi/g17/object.rs

@@ -459,6 +459,11 @@ struct Retained {
     prefix: usize,
     cpu: CpuMap,
     poisoned: bool,
+    /// Return order: reuse takes the object returned longest ago.
+    order: u64,
+    /// Firmware progress after which the previous owner's references are gone
+    /// (see [`super::channel::control_fence`]).
+    fence: u64,
 }
 
 struct PoolInner {
@@ -472,6 +477,10 @@ struct PoolInner {
     kept_bytes: usize,
     /// Kept objects still owned by their users, whose list entries are reserved.
     kept_reserved: usize,
+    /// Objects returned so far, ordering the reuse cache.
+    returns: u64,
+    /// Reuses that had to take an object whose fence the firmware had not met yet.
+    fence_overruns: u64,
 }
 
 /// Retained objects past this many are still retained, with one warning.
@@ -554,7 +563,7 @@ impl Pool {
             pin_init!(Self {
                 inner <- kernel::new_mutex!(PoolInner {
                     returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(),
-                    kept_bytes: 0, kept_reserved: 0,
+                    kept_bytes: 0, kept_reserved: 0, returns: 0, fence_overruns: 0,
                 }, "g17::object::Pool"),
             }),
             GFP_KERNEL,
@@ -563,6 +572,12 @@ impl Pool {
 
     /// Allocates or reuses a matching object. Capacity pressure leaves a fresh
     /// mapping uncached, preserving allocation and submission concurrency.
+    ///
+    /// The firmware may still follow a previous owner's pointers into a returned object
+    /// until it has processed that owner's release. Reuse takes the matching object
+    /// returned longest ago once the firmware consumed a control record published after
+    /// its return; a younger one is reused only when the family has no room for a fresh
+    /// cached object, because an uncached one is unmapped as soon as its owner drops it.
     pub(crate) fn allocate(
         self: &kernel::sync::Arc<Self>,
         alloc: &Allocator<'_>,
@@ -574,22 +589,40 @@ impl Pool {
     ) -> Result<PooledObject> {
         let family_index = family as usize;
         let begin = family_index * POOL_SLOTS;
+        let witnessed = super::channel::control_witnessed();
         let reused = {
             let mut inner = self.inner.lock();
             if inner.stopped {
                 return Err(ENODEV);
             }
-            let slot = inner.returned[begin..begin + POOL_SLOTS]
-                .iter_mut()
-                .find(|entry| {
-                    entry.as_ref().is_some_and(|entry| {
-                        !entry.poisoned
-                            && entry.object.size() == size
-                            && entry.prefix == prefix
-                            && entry.cpu == cpu
-                    })
-                });
-            slot.and_then(Option::take)
+            let room = inner.reserved[family_index] < POOL_SLOTS
+                && inner.bytes.checked_add(size).is_some_and(|bytes| bytes <= POOL_BYTES);
+            let mut oldest: Option<(usize, u64, bool)> = None;
+            for (index, entry) in inner.returned[begin..begin + POOL_SLOTS].iter().enumerate() {
+                let Some(entry) = entry.as_ref().filter(|entry| {
+                    !entry.poisoned
+                        && entry.object.size() == size
+                        && entry.prefix == prefix
+                        && entry.cpu == cpu
+                }) else {
+                    continue;
+                };
+                let ready = entry.fence <= witnessed;
+                if oldest.is_none_or(|(_, order, best_ready)| {
+                    (ready && !best_ready) || (ready == best_ready && entry.order < order)
+                }) {
+                    oldest = Some((begin + index, entry.order, ready));
+                }
+            }
+            match oldest {
+                Some((index, _, ready)) if ready || !room => {
+                    if !ready {
+                        inner.fence_overruns += 1;
+                    }
+                    inner.returned[index].take()
+                }
+                _ => None,
+            }
         };
         let lease = || CacheLease {
             pool: self.clone(),
@@ -671,13 +704,18 @@ impl Pool {
             return;
         }
         let begin = family * POOL_SLOTS;
+        inner.returns += 1;
+        let order = inner.returns;
+        let fence = super::channel::control_fence();
         for slot in &mut inner.returned[begin..begin + POOL_SLOTS] {
             if slot.is_none() {
                 *slot = Some(Retained {
                     object,
                     prefix: lease.prefix,
                     cpu: lease.cpu,
                     poisoned: lease.poisoned,
+                    order,
+                    fence,
                 });
                 return;
             }
@@ -776,6 +814,10 @@ impl Pool {
                      entries.iter().flatten().filter(|entry| entry.poisoned).count())?;
         }
         writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?;
+        let witnessed = super::channel::control_witnessed();
+        writeln!(f, "cached_objects_fenced waiting={} overruns={}",
+                 inner.returned.iter().flatten().filter(|entry| entry.fence > witnessed).count(),
+                 inner.fence_overruns)?;
         writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes)
     }
 }

drivers/gpu/drm/asahi/g17/runtime.rs

@@ -489,6 +489,7 @@ impl Registry {
         fence(Ordering::Acquire);
         let consumer = init.control_consumer()?.load(Ordering::Relaxed);
         let producer = init.control_producer()?.load(Ordering::Relaxed);
+        super::channel::observe_control(consumer, producer);
         let mut freed = 0;
         for mut queue in self
             .compute
@@ -722,7 +723,7 @@ impl Backend {
                 // Creators hold the device mutex, so neither check above can
                 // have changed; the slot only fails on a broken invariant.
                 let slot = admission.reserve_queue()?;
-                firmware.uat.new_execution_context(vm).map(|execution| (execution, slot))
+                firmware.uat.new_execution_context(vm, super::channel::control_witnessed()).map(|execution| (execution, slot))
             } else {
                 Err(ENOSPC)
             };

drivers/gpu/drm/asahi/g17/runtime/teardown.rs

@@ -202,6 +202,7 @@ impl Firmware {
         teardown.publish_ready(&mut service, recovery, super::now_ns())?;
         let consumer = self.init.control_consumer()?.load(Ordering::Relaxed);
         let producer = self.init.control_producer()?.load(Ordering::Relaxed);
+        crate::g17::channel::observe_control(consumer, producer);
         let mut consumed = false;
         while teardown
             .take_consumed(consumer, producer, &mut service)?