drm/asahi: kill faulted G17 contexts before freeing their render work

drivers/gpu/drm/asahi/g17/context.rs

@@ -280,6 +280,28 @@ impl Context {
             _work: self.work.clone(),
         }
     }
+    /// Called once the firmware consumed this context's scheduler release. If its work list
+    /// still links a node, the firmware keeps referencing that node: on Neo a render command
+    /// that ran behind a failed compute command is never unlinked, and a later context given
+    /// the same work storage (the object cache hands it out again) made the firmware recover
+    /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether
+    /// it was linked.
+    pub(crate) fn retain_linked_work(&self, pool: &Arc<Pool>) -> bool {
+        // An unreadable head counts as linked: retaining is the safe side.
+        let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]);
+        if linked {
+            self.work.lock().object.retain_until_stop(pool);
+        }
+        linked
+    }
+
+    /// The firmware may still walk this context's work list into memory it names: a node is
+    /// linked and the scheduler page shows no kill. Unreadable state counts as linked.
+    pub(crate) fn firmware_may_link(&self) -> bool {
+        let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed);
+        must_retain(self.work_head().ok(), self.work_head_va(), killed)
+    }
+
     pub(crate) fn work_head_va(&self) -> u64 {
         self.work_base + (UAT_PGSZ + 0x400) as u64
     }
@@ -463,10 +485,26 @@ impl QosIds {
     }
 }
 
+/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or
+/// an unreadable head) and no acknowledged kill.
+fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool {
+    killed == 0 && head.is_none_or(|head| head != [0, head_va])
+}
+
 #[kunit_tests(asahi_g17_qos_capacity)]
 mod capacity_tests {
     use super::*;
 
+    #[test]
+    fn linked_unkilled_work_is_retained() {
+        let va = 0xfffffc20_0000_4400;
+        assert!(!must_retain(Some([0, va]), va, 0));
+        assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0));
+        assert!(must_retain(None, va, 0));
+        assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1));
+        assert!(!must_retain(None, va, 1));
+    }
+
     #[test]
     fn failed_client_construction_returns_every_qos_id() {
         let ids = QosIds::new().unwrap();

drivers/gpu/drm/asahi/g17/event.rs

@@ -75,6 +75,7 @@ fn batch<H: Host>(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result {
             Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?,
             Event::ContextKilled(cookie) => host.context_killed(cookie),
             Event::ComputeFault { qid, stamp } => {
+                crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0);
                 host.compute_fault(qid, stamp);
                 // Its queue retires or settles its quarantine on this pass.
                 masks[usize::from(qid) / 64] |= 1 << (qid % 64);

drivers/gpu/drm/asahi/g17/kick.rs

@@ -145,6 +145,11 @@ impl QueueIds {
         self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2
     }
 
+    /// Whether a render pair not yet built can still reserve both of its identities.
+    pub(crate) fn render_pair_fits(&self) -> bool {
+        QID_COUNT - self.floor - self.leased >= 2
+    }
+
     /// Whether a fresh compute identity fits beside the render reserve of
     /// `queues` open logical queues, without reserving it.
     pub(crate) fn compute_fits(&self, queues: usize) -> bool {

drivers/gpu/drm/asahi/g17/object.rs

@@ -466,8 +466,17 @@ struct PoolInner {
     reserved: [usize; POOL_FAMILIES],
     bytes: usize,
     stopped: bool,
+    /// Objects the firmware may still reference after their owner dropped them, kept mapped
+    /// and out of reuse until firmware stop (see [`PooledObject::retain_until_stop`]).
+    kept: KVec<KernelObject>,
+    kept_bytes: usize,
+    /// Kept objects still owned by their users, whose list entries are reserved.
+    kept_reserved: usize,
 }
 
+/// Retained objects past this many are still retained, with one warning.
+const KEPT_WARN: usize = 256;
+
 /// Per-device cache-safe object recycling. Every converted mapping reserves
 /// one return slot before its page-table attributes change. Active leases and
 /// poisoned mappings count against the same capacity as reusable objects.
@@ -490,6 +499,7 @@ struct CacheLease {
 pub(crate) struct PooledObject {
     object: core::mem::ManuallyDrop<KernelObject>,
     lease: Option<CacheLease>,
+    keep: Option<kernel::sync::Arc<Pool>>,
 }
 
 impl core::ops::Deref for PooledObject {
@@ -503,11 +513,28 @@ impl core::ops::DerefMut for PooledObject {
         &mut self.object
     }
 }
+impl PooledObject {
+    /// The firmware may still reference this object after its owner drops it: keep it mapped
+    /// and never hand it out again before firmware stop. It leaves the reuse cache, so it
+    /// does not take cache capacity. Call from sleepable context: this reserves the list
+    /// entry, so the drop, wherever it happens, does not allocate.
+    pub(crate) fn retain_until_stop(&mut self, pool: &kernel::sync::Arc<Pool>) {
+        if self.keep.is_none() {
+            pool.reserve_kept();
+            self.keep = Some(pool.clone());
+        }
+    }
+}
+
 impl Drop for PooledObject {
     fn drop(&mut self) {
         // SAFETY: This destructor is the only operation that moves `object`
         // out; ManuallyDrop prevents a second implicit drop of the field.
         let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) };
+        if let Some(pool) = self.keep.take() {
+            pool.keep_object(object, self.lease.take());
+            return;
+        }
         match self.lease.take() {
             Some(lease) => lease.pool.return_object(object, &lease),
             None => drop(object),
@@ -523,7 +550,10 @@ impl Pool {
         }
         kernel::sync::Arc::pin_init(
             pin_init!(Self {
-                inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"),
+                inner <- kernel::new_mutex!(PoolInner {
+                    returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(),
+                    kept_bytes: 0, kept_reserved: 0,
+                }, "g17::object::Pool"),
             }),
             GFP_KERNEL,
         )
@@ -572,6 +602,7 @@ impl Pool {
             return Ok(PooledObject {
                 object: core::mem::ManuallyDrop::new(entry.object),
                 lease: Some(lease()),
+                keep: None,
             });
         }
         let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?;
@@ -596,6 +627,7 @@ impl Pool {
         let mut result = PooledObject {
             object: core::mem::ManuallyDrop::new(object),
             lease: reserved.then(lease),
+            keep: None,
         };
         if !reserved {
             return Ok(result);
@@ -655,10 +687,58 @@ impl Pool {
         core::mem::forget(object);
     }
 
+    /// Room for one more kept object, taken by the matching `keep_object`. A failed
+    /// allocation is reported by that drop, which then leaks the object instead.
+    fn reserve_kept(&self) {
+        let mut inner = self.inner.lock();
+        let pending = inner.kept_reserved + 1;
+        if inner.kept.capacity() - inner.kept.len() < pending
+            && inner.kept.reserve(pending, GFP_KERNEL).is_err()
+        {
+            return;
+        }
+        inner.kept_reserved = pending;
+    }
+
+    fn keep_object(&self, object: KernelObject, lease: Option<CacheLease>) {
+        let mut inner = self.inner.lock();
+        inner.kept_reserved = inner.kept_reserved.saturating_sub(1);
+        if let Some(lease) = &lease {
+            // The object leaves the reuse cache and its capacity.
+            inner.reserved[lease.family as usize] -= 1;
+            inner.bytes -= object.size();
+        }
+        if inner.stopped {
+            drop(inner);
+            drop(object);
+            return;
+        }
+        let size = object.size();
+        if inner.kept.len() == KEPT_WARN {
+            pr_warn!("G17 keeps {} firmware-linked objects until the GPU firmware restarts\n", KEPT_WARN);
+        }
+        // retain_until_stop reserved the entry; this never allocates.
+        match inner.kept.push_within_capacity(object) {
+            Ok(()) => inner.kept_bytes += size,
+            Err(error) => {
+                // Never unmap an object the firmware may still reference.
+                pr_err!("G17 could not record a firmware-linked object; leaking it\n");
+                core::mem::forget(error.0);
+            }
+        }
+    }
+
     /// Releases retained mappings after both firmware processors have stopped.
     /// Active objects returned later observe the same stop witness.
     pub(crate) fn stop_after_firmware(&self) {
-        self.inner.lock().stopped = true;
+        let kept = {
+            let mut inner = self.inner.lock();
+            inner.stopped = true;
+            inner.kept_bytes = 0;
+            core::mem::take(&mut inner.kept)
+        };
+        // Mapping teardown must not run while the pool mutex is held.
+        drop(kept);
         for index in 0..POOL_FAMILIES * POOL_SLOTS {
             let entry = {
                 let mut inner = self.inner.lock();
@@ -684,6 +764,7 @@ impl Pool {
                      inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(),
                      entries.iter().flatten().filter(|entry| entry.poisoned).count())?;
         }
-        writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)
+        writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?;
+        writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes)
     }
 }