drm/asahi: G17 leak fix, render notifications and hop fixes from aurora (stacked on #229)

drivers/gpu/drm/asahi/g17/context.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Logical queue ownership of scheduler, QoS and work-state identities.
@@ -114,6 +114,9 @@
     scheduler: Arc<KernelObject>,
     scheduler_generation: u32,
     published: AtomicBool,
+    /// A kill of this context was published, through a compute queue or on its render pairs'
+    /// behalf. A context is killed at most once.
+    kill_published: AtomicBool,
     /// Advisory only: publication still checks the physical binding under the
     /// device mutex. False skips an inline attempt known to need admission.
     compute_binding_hint: AtomicBool,
@@ -197,6 +200,7 @@
             scheduler,
             scheduler_generation,
             published: AtomicBool::new(false),
+            kill_published: AtomicBool::new(false),
             compute_binding_hint: AtomicBool::new(false),
             render_seen: AtomicBool::new(false),
             qos,
@@ -280,6 +284,28 @@
             _work: self.work.clone(),
         }
     }
+    /// Called once the firmware consumed this context's scheduler release. If its work list
+    /// still links a node, the firmware keeps referencing that node: on Neo a render command
+    /// that ran behind a failed compute command is never unlinked, and a later context given
+    /// the same work storage (the object cache hands it out again) made the firmware recover
+    /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether
+    /// it was linked.
+    pub(crate) fn retain_linked_work(&self, pool: &Arc<Pool>) -> bool {
+        // An unreadable head counts as linked: retaining is the safe side.
+        let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]);
+        if linked {
+            self.work.lock().object.retain_until_stop(pool);
+        }
+        linked
+    }
+
+    /// The firmware may still walk this context's work list into memory it names: a node is
+    /// linked and the scheduler page shows no kill. Unreadable state counts as linked.
+    pub(crate) fn firmware_may_link(&self) -> bool {
+        let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed);
+        must_retain(self.work_head().ok(), self.work_head_va(), killed)
+    }
+
     pub(crate) fn work_head_va(&self) -> u64 {
         self.work_base + (UAT_PGSZ + 0x400) as u64
     }
@@ -294,6 +320,16 @@
         Ok(head)
     }
 
+    /// Record that a kill of this context was published (see `kill_published`).
+    pub(crate) fn note_kill_published(&self) {
+        self.kill_published.store(true, Ordering::Release);
+    }
+
+    /// A kill of this context was published.
+    pub(crate) fn kill_published(&self) -> bool {
+        self.kill_published.load(Ordering::Acquire)
+    }
+
     pub(crate) fn is_published(&self) -> bool {
         self.published.load(Ordering::Acquire)
     }
@@ -463,10 +499,26 @@
     }
 }
 
+/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or
+/// an unreadable head) and no acknowledged kill.
+fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool {
+    killed == 0 && head.is_none_or(|head| head != [0, head_va])
+}
+
 #[kunit_tests(asahi_g17_qos_capacity)]
 mod capacity_tests {
     use super::*;
 
+    #[test]
+    fn linked_unkilled_work_is_retained() {
+        let va = 0xfffffc20_0000_4400;
+        assert!(!must_retain(Some([0, va]), va, 0));
+        assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0));
+        assert!(must_retain(None, va, 0));
+        assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1));
+        assert!(!must_retain(None, va, 1));
+    }
+
     #[test]
     fn failed_client_construction_returns_every_qos_id() {
         let ids = QosIds::new().unwrap();

drivers/gpu/drm/asahi/g17/event.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Event service ordering under the device mutex. A restart ends its role's batch; a worker
@@ -36,6 +36,8 @@
     fn retain_render_completions(&mut self, masks: [u64; 2]);
     fn complete_compute(&mut self, masks: [u64; 2]) -> Result;
     fn context_killed(&mut self, cookie: u64);
+    /// The firmware failed a compute command (see [`Event::ComputeFault`]).
+    fn compute_fault(&mut self, qid: u8, stamp: u32);
 }
 
 /// Work requested after releasing the device mutex.
@@ -72,6 +74,12 @@
             }
             Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?,
             Event::ContextKilled(cookie) => host.context_killed(cookie),
+            Event::ComputeFault { qid, stamp } => {
+                crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0);
+                host.compute_fault(qid, stamp);
+                // Its queue retires or settles its quarantine on this pass.
+                masks[usize::from(qid) / 64] |= 1 << (qid % 64);
+            }
             Event::Unknown(_) => {}
         }
         Ok(true)

drivers/gpu/drm/asahi/g17/fw/channels.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Device-control records and work-ring slots.
@@ -136,6 +136,9 @@
     TvbGrow(TvbGrowRequest),
     FreeListGrow(FreeListCompletion),
     ContextKilled(u64),
+    /// The firmware failed a compute command of queue `qid` and stores no completion stamp for
+    /// it: `stamp` is the value that command would have stored.
+    ComputeFault { qid: u8, stamp: u32 },
     Unknown(u32),
 }
 
@@ -145,6 +148,8 @@
     const TVB_GROW: u32 = 6;
     const FREE_LIST_GROW: u32 = 13;
     const CONTEXT_KILLED: u32 = 12;
+    const COMPUTE_FAULT: u32 = 7;
+    const COMPUTE_QIDS: u32 = 128;
     const DIAGNOSTIC_PAIRS_MAX: u16 = 5;
     const TVB_SLOTS: u32 = 127;
     const SUBPIPES: u32 = 64;
@@ -198,6 +203,19 @@
                     halt_count: record.halt_count,
                 }))
             }
+            Self::COMPUTE_FAULT => {
+                // SAFETY: Every constructor initializes all integer words.
+                let words = unsafe { self.words };
+                // Only compute faults are known to use this type: any other layout stays an
+                // unknown event, consumed and ignored as before.
+                if words[3] >= Self::COMPUTE_QIDS || words[4] & 0xff != 0 {
+                    return Ok(Event::Unknown(Self::COMPUTE_FAULT));
+                }
+                Ok(Event::ComputeFault {
+                    qid: words[3] as u8,
+                    stamp: words[4],
+                })
+            }
             Self::CONTEXT_KILLED => {
                 // SAFETY: Every constructor initializes all integer words.
                 let words = unsafe { self.words };
@@ -525,3 +543,26 @@
         }
     }
 }
+
+#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_event_records))]
+mod tests {
+    use super::*;
+
+    fn record(words: &[u32]) -> EventRecord {
+        let mut all = [0u32; EVENT_WORDS];
+        all[..words.len()].copy_from_slice(words);
+        EventRecord::from_words(all)
+    }
+
+    #[test]
+    fn a_compute_fault_names_its_queue_and_stamp() {
+        let event = record(&[7, 1, 2, 0x39, 0x300, 0xffff_fc00, 1]).decode();
+        assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 }));
+    }
+
+    #[test]
+    fn an_unrecognized_type_7_layout_stays_unknown() {
+        assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7)));
+        assert_eq!(record(&[7, 1, 2, 9, 0x301]).decode(), Ok(Event::Unknown(7)));
+    }
+}

drivers/gpu/drm/asahi/g17/fw/render.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Render work descriptors.
@@ -1023,7 +1023,11 @@
     /// QID of the tiling queue.
     pub(crate) qid: u32,
     pub(crate) unk_8be: u32,
-    pub(crate) unk_8c2: u32,
+    /// Non-zero on the first kick of a queue pair, like the fragment descriptor's word at the
+    /// same position. Set on every kick, the firmware also notifies the host of each pass's
+    /// tiling completion, which no retirement needs. The 25G83 firmware keeps it set on every
+    /// kick until this is measured there.
+    pub(crate) first_kick: u32,
     /// Completion value of the tiling kick (see
     /// [`KickTimestamp::completion_value`]).
     pub(crate) completion: u64,
@@ -1073,7 +1077,6 @@
     const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64;
     const UNK_876: u32 = u32::MAX;
     const UNK_892: u32 = 1;
-    const UNK_8C2: u32 = 1;
     const UNK_932: u8 = 0x44;
     const UNK_93C: u8 = 1;
     const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91];
@@ -1127,7 +1130,10 @@
         self.aux_stamp_va = args.ta.aux_stamp_va;
         self.stamp_value_2 = stamp_value;
         self.qid = args.ta.qid.into();
-        self.unk_8c2 = Self::UNK_8C2;
+        self.first_kick = match args.programs == RegisterPrograms::V25G83 {
+            true => 1,
+            false => u32::from(args.ordinal == 0),
+        };
         self.current_id = resources.ta_ids.current.into();
         self.ordinal = args.ordinal as u32;
         self.timestamp_start_va = resources.timestamp_va;

drivers/gpu/drm/asahi/g17/job.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Immutable submissions and the DRM scheduler boundary for independent engines.
@@ -293,6 +293,8 @@
     fn cancel(&self, packet: Arc<Packet>);
     /// Transfer a timeout to an outstanding off-lock graph transaction, if any.
     fn defer_timeout(&self, packet: &Arc<Packet>, renewals: u8) -> bool;
+    /// A long job of another queue is running beside `packet`.
+    fn long_job_beside(&self, _packet: &Arc<Packet>) -> bool { false }
     /// Return true only after taking publication ownership and recording its result.
     fn try_early(&self, packet: &Arc<Packet>) -> bool;
     fn count_backlog(&self, packet: &Packet);
@@ -346,11 +348,11 @@
         ) {
             return true;
         }
-        // Other work retiring means the GPU is shared, not hung.
-        if super::timeout::renew_shared(
-            job.packet.is_published(), super::recovery::since_retired(super::now_ns()),
-            &mut job.shared_renewals,
-        ) {
+        // Other work retiring means the GPU is shared, not hung. So does another long job
+        // running beside a long job, though neither retires until it ends.
+        let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns()))
+            || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet));
+        if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) {
             return true;
         }
         let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals);

drivers/gpu/drm/asahi/g17/kick.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Physical kick-ring backing, queue identities and entry publication.
@@ -43,7 +43,7 @@
 /// The render reserve with every execution context open.
 const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize);
 /// Compute channels that stay reachable with every execution context open.
-#[cfg(CONFIG_KUNIT)]
+#[cfg(CONFIG_KUNIT = "y")]
 pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE;
 /// Upper bound on distinct physical compute channels named by one kick.
 pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT;
@@ -145,6 +145,11 @@
         self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2
     }
 
+    /// Whether a render pair not yet built can still reserve both of its identities.
+    pub(crate) fn render_pair_fits(&self) -> bool {
+        QID_COUNT - self.floor - self.leased >= 2
+    }
+
     /// Whether a fresh compute identity fits beside the render reserve of
     /// `queues` open logical queues, without reserving it.
     pub(crate) fn compute_fits(&self, queues: usize) -> bool {

drivers/gpu/drm/asahi/g17/mod.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! G17 GPU support.
@@ -65,6 +65,7 @@
 use core::sync::atomic::{
     AtomicBool,
     AtomicU32,
+    AtomicU64,
     Ordering, //
 };
 
@@ -369,7 +370,7 @@
         if !is_events_message(ep, msg) {
             return false;
         }
-        data.shared.queue_events();
+        data.shared.queue_event_records();
         true
     }
 
@@ -857,6 +858,11 @@
     compute_demand: AtomicU32,
     #[pin]
     work: Work<Shared, EVENT_WORK>,
+    /// The next event pass runs every housekeeping step: set by every [`Self::queue_events`]
+    /// request, not by the firmware's event notification ([`Self::queue_event_records`]).
+    housekeeping: AtomicBool,
+    /// When an event pass last ran every housekeeping step, in ns.
+    housekeeping_ns: AtomicU64,
     #[pin]
     poll: DelayedWork<Shared, POLL_WORK>,
     #[pin]
@@ -989,8 +995,17 @@
         self.port.store(core::ptr::null_mut(), Ordering::Release);
     }
 
-    /// Queues the event worker. Callable from any context.
+    /// Queues the event worker for a pass that runs every housekeeping step. Callable from any
+    /// context.
     fn queue_events(self: &Arc<Self>) {
+        self.housekeeping.store(true, Ordering::Release);
+        self.queue_event_records();
+    }
+
+    /// Queues the event worker for firmware records alone: completions, or records that arrived
+    /// during a pass. The pass skips the housekeeping that nothing needs (see
+    /// `runtime_event_turn`). Callable from any context.
+    fn queue_event_records(self: &Arc<Self>) {
         // Already queued if this fails, which is just as good.
         let _ = workqueue::system_highpri().enqueue::<Arc<Shared>, EVENT_WORK>(self.clone());
     }
@@ -1170,6 +1185,8 @@
                 renders,
                 compute_demand: AtomicU32::new(0),
                 work <- new_work!("g17::Shared::work"),
+                housekeeping: AtomicBool::new(true),
+                housekeeping_ns: AtomicU64::new(0),
                 poll <- new_delayed_work!("g17::Shared::poll"),
                 idle <- new_delayed_work!("g17::Shared::idle"),
                 grow <- new_work!("g17::Shared::grow"),