drm/asahi: g17: M5: drop repeat uncached reads from submission, render publication and the event pass

drivers/gpu/drm/asahi/g17/freelist.rs

@@ -1371,6 +1371,11 @@ impl Lease {
     }
 
     /// Observes all older release records passed by one consumer snapshot.
+    /// A sent release whose consumption [`Self::observe_releases`] has yet to witness.
+    pub(crate) fn awaits_release(&self) -> bool {
+        self.previous_release.is_some() || matches!(self.phase, LeasePhase::ReleaseSent(_))
+    }
+
     pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result {
         if let Some(cursor) = self.previous_release {
             if control_consumed(consumer, producer, cursor)? {

drivers/gpu/drm/asahi/g17/initdata.rs

@@ -205,6 +205,9 @@ pub(super) struct InitData {
     compute_scheduler: Option<ComputeScheduler>,
     compute_active: core::cell::Cell<bool>,
     event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2],
+    /// M5: the submission-history geometry (period, slot count) read under the firmware's
+    /// generation, with that generation.
+    history_geometry: core::cell::Cell<Option<(u32, u64, u64)>>,
     /// Completion-ring objects; a family sets only the rings it has.
     completion_rings: [Option<usize>; 4],
     stamps: [usize; 2],
@@ -308,6 +311,7 @@ impl InitData {
             compute_scheduler: None,
             compute_active: core::cell::Cell::new(false),
             event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2],
+            history_geometry: core::cell::Cell::new(None),
             completion_rings: [None; 4],
             stamps: [0; 2],
             render_pages: None,
@@ -691,10 +695,37 @@ impl InitData {
         Ok(next != consumer && next != consumer_b)
     }
 
+    /// M5 compute: the shared ring's cursors when it has room for one record, as
+    /// [`Self::work_ready`] judges it. The same critical section may publish against them
+    /// ([`Self::publish_work_at`]): the firmware's consumers only advance, and the producer
+    /// is the device mutex holder's.
+    pub(super) fn m5_compute_room(&self, priority: u8) -> Result<Option<[u32; 3]>> {
+        use super::fw::queue::DataMaster;
+        if !self.m5 {
+            return Err(EINVAL);
+        }
+        let cursors = self.work_cursors(priority, DataMaster::Compute)?;
+        Ok((self.outer_capacity(priority, cursors)? >= 1).then_some(cursors))
+    }
+
+    /// M5 render: the shared ring's cursors when it has room for a pair, as
+    /// [`Self::work_ready`] judges it, for the pair's publication in the same hold
+    /// ([`Self::m5_work_pair`]).
+    pub(super) fn m5_render_room(&self, priority: u8) -> Result<Option<[u32; 3]>> {
+        use super::fw::queue::DataMaster;
+        if !self.m5 {
+            return Err(EINVAL);
+        }
+        let cursors = self.work_cursors(priority, DataMaster::Compute)?;
+        Ok((self.outer_capacity(priority, cursors)? >= 2).then_some(cursors))
+    }
+
+    /// `cursors`: an M5 compute room check's sample ([`Self::m5_compute_room`]), else read here.
     fn work_publication<'a>(
         &'a self,
         priority: u8,
         record: &'a super::fw::channels::WorkSlot,
+        cursors: Option<[u32; 3]>,
     ) -> Result<WorkPublication<'a>> {
         use super::fw::{channels, queue::DataMaster};
         let engine = match record.data_master {
@@ -706,7 +737,11 @@ impl InitData {
             _ => return Err(EINVAL),
         };
         let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?;
-        let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?;
+        let [consumer, consumer_b, producer] = match cursors {
+            Some(cursors) if self.m5 && engine == DataMaster::Compute => cursors,
+            Some(_) => return Err(EINVAL),
+            None => self.work_cursors(priority, engine)?,
+        };
         if self.m5 && engine == DataMaster::Compute
             && self.outer_capacity(priority, [consumer, consumer_b, producer])? < 1 {
             return Err(EBUSY);
@@ -736,6 +771,21 @@ impl InitData {
         })
     }
 
+    /// The host producer of the shared M5 ring alone (the last of [`Self::work_cursors`]).
+    pub(super) fn work_producer(&self, priority: u8) -> Result<u32> {
+        use super::fw::{channels, queue::DataMaster};
+        let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?;
+        let producer = self
+            .object(self.cluster)?
+            .word(self.state_offset(Role::Primary) + abi::work_state_offset(channel)
+                + abi::ChannelState::PRODUCER)?
+            .load(Ordering::Relaxed);
+        if producer >= channels::WORK_RING_SLOTS {
+            return Err(EINVAL);
+        }
+        Ok(producer)
+    }
+
     /// Two firmware consumers and the host producer, sampled in that order.
     pub(super) fn work_cursors(
         &self,
@@ -769,16 +819,30 @@ impl InitData {
         priority: u8,
         record: &super::fw::channels::WorkSlot,
     ) -> Result<u32> {
-        Ok(self.work_publication(priority, record)?.commit())
+        self.publish_work_at(priority, record, None)
+    }
+
+    /// [`Self::publish_work`] against the cursors of a room check made earlier under the same
+    /// hold of the device mutex ([`Self::m5_compute_room`]), or a fresh read for None.
+    pub(super) fn publish_work_at(
+        &self,
+        priority: u8,
+        record: &super::fw::channels::WorkSlot,
+        cursors: Option<[u32; 3]>,
+    ) -> Result<u32> {
+        Ok(self.work_publication(priority, record, cursors)?.commit())
     }
 
     /// Reserve both common-ring slots before touching either host port. The
     /// returned views retain the graph borrow until the pair is published.
+    /// `cursors`: a room check's sample from the same hold ([`Self::m5_render_room`]),
+    /// else read here.
     pub(super) fn m5_work_pair(
         &self,
         priority: u8,
         fragment: &super::fw::channels::WorkSlot,
         tiling: &super::fw::channels::WorkSlot,
+        cursors: Option<[u32; 3]>,
     ) -> Result<work_pair::Publication<'_>> {
         use super::fw::{channels, queue::DataMaster};
         if !self.m5
@@ -787,7 +851,10 @@ impl InitData {
         {
             return Err(EINVAL);
         }
-        let cursors = self.work_cursors(priority, DataMaster::Compute)?;
+        let cursors = match cursors {
+            Some(cursors) => cursors,
+            None => self.work_cursors(priority, DataMaster::Compute)?,
+        };
         if self.outer_capacity(priority, cursors)? < 2 { return Err(EBUSY); }
         let window = work_pair::Window::new(cursors)?;
         let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?;
@@ -829,14 +896,14 @@ impl InitData {
             return Err(EINVAL);
         }
         if self.m5 {
-            let pair = self.m5_work_pair(priority, fragment, tiling)?;
+            let pair = self.m5_work_pair(priority, fragment, tiling, None)?;
             pair.prepare();
             commit_accounting()?;
             publish_late_tiling();
             return Ok(if deferred { pair.next() } else { pair.publish() });
         }
-        let fragment = self.work_publication(priority, fragment)?;
-        let tiling = self.work_publication(priority, tiling)?;
+        let fragment = self.work_publication(priority, fragment, None)?;
+        let tiling = self.work_publication(priority, tiling, None)?;
         fragment.copy_record();
         commit_accounting()?;
         fence(Ordering::SeqCst);
@@ -891,16 +958,18 @@ impl InitData {
 
     /// Release a retained pair after its control transaction. Revalidate the original producer
     /// witnesses before either copy; both bodies precede the fragment and tiling publications.
+    /// `cursors`: M5, as for [`Self::m5_work_pair`].
     pub(super) fn publish_retained_work_pair(
         &self,
         priority: u8,
         fragment: &super::fw::channels::WorkSlot,
         tiling: &super::fw::channels::WorkSlot,
         next: [u32; 2],
+        cursors: Option<[u32; 3]>,
     ) -> Result {
         use super::fw::queue::DataMaster;
         if self.m5 {
-            let pair = self.m5_work_pair(priority, fragment, tiling)?;
+            let pair = self.m5_work_pair(priority, fragment, tiling, cursors)?;
             if pair.next() != next { return Err(EBUSY); }
             pair.publish();
             return Ok(());
@@ -1936,6 +2005,8 @@ impl super::recovery::Memory for InitData {
     }
 
     fn transition(&self, expected: u32, next: u32) -> Result {
+        // A restarted firmware may lay its submission history out anew.
+        self.history_geometry.set(None);
         let state = self
             .object(self.cluster)?
             .word(self.recovery_state_offset())?;

drivers/gpu/drm/asahi/g17/initdata/m5.rs

@@ -55,13 +55,22 @@ impl InitData {
         for _ in 0..64 {
             let generation = private.word(base + 0x74c)?.load(Ordering::Acquire);
             if generation & 1 != 0 { core::hint::spin_loop(); continue; }
-            let config = private.read_words::<4>(base + 0x73c)?;
-            fence(Ordering::Acquire);
-            if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) {
-                core::hint::spin_loop(); continue;
-            }
-            let period = u64::from(config[0]) | u64::from(config[1]) << 32;
-            let count = u64::from(config[2]) | u64::from(config[3]) << 32;
+            // The firmware moves the generation around every change of the geometry: while it
+            // reads as it did, the geometry read under it stands, without four more loads.
+            let (period, count) = match self.history_geometry.get() {
+                Some((seen, period, count)) if seen == generation => (period, count),
+                _ => {
+                    let config = private.read_words::<4>(base + 0x73c)?;
+                    fence(Ordering::Acquire);
+                    if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) {
+                        core::hint::spin_loop(); continue;
+                    }
+                    let period = u64::from(config[0]) | u64::from(config[1]) << 32;
+                    let count = u64::from(config[2]) | u64::from(config[3]) << 32;
+                    self.history_geometry.set(Some((generation, period, count)));
+                    (period, count)
+                }
+            };
             if period == 0 || count == 0 || count > 256 { return Err(EINVAL); }
             let (counter, frequency) = crate::g17::qos::clock();
             let ticks = crate::g17::qos::time_from_host(counter, frequency)?;

drivers/gpu/drm/asahi/g17/mod.rs

@@ -798,11 +798,13 @@ impl Firmware {
         commit_accounting: impl FnOnce() -> Result,
         publish_inner: impl FnOnce(),
         deferred: bool,
+        cursors: Option<[u32; 3]>,
     ) -> Result<[u32; 2]> {
         let mut commit_accounting = Some(commit_accounting);
         let mut publish_inner = Some(publish_inner);
+        let mut cursors = cursors;
         for attempt in 0..2 {
-            let pair = self.init.m5_work_pair(priority, &records[0], &records[1])?;
+            let pair = self.init.m5_work_pair(priority, &records[0], &records[1], cursors.take())?;
             pair.prepare();
             if let Some(commit) = commit_accounting.take() { commit()?; }
             match self.bridge.regs.submit_m5_work(submission) {

drivers/gpu/drm/asahi/g17/object.rs

@@ -328,6 +328,17 @@ impl KernelObject {
     pub(crate) fn read_words<const N: usize>(&self, offset: usize) -> Result<[u32; N]> {
         let base = self.aligned::<[u32; N]>(offset)?.cast::<u32>();
         let mut words = [0u32; N];
+        // Most views are uncached: where the copy is 8-byte aligned, load two words at a time.
+        if N % 2 == 0 && base as usize % 8 == 0 {
+            let pairs = base.cast::<u64>();
+            for (i, pair) in words.chunks_exact_mut(2).enumerate() {
+                // SAFETY: As below; the 8-byte loads cover the same aligned `N` words.
+                let bytes = unsafe { pairs.add(i).read_volatile() }.to_ne_bytes();
+                pair[0] = u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
+                pair[1] = u32::from_ne_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
+            }
+            return Ok(words);
+        }
         for (i, word) in words.iter_mut().enumerate() {
             // SAFETY: `aligned()` checked that all `N` words are within the object and aligned.
             *word = unsafe { base.add(i).read_volatile() };

drivers/gpu/drm/asahi/g17/queue/compute.rs

@@ -2408,7 +2408,17 @@ impl Graph {
         Ok(object.gpu_va() + start as u64)
     }
     fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result<u32> {
-        let cursors = self.cursors()?;
+        self.append_to(self.cursors()?, expected, address, before_publish)
+    }
+    /// Appends against `cursors`: a fresh read, or an M5 publication's snapshot whose
+    /// producer the publication advanced itself (see `publish_with_native_waits`).
+    fn append_to(
+        &self,
+        cursors: Cursors,
+        expected: u32,
+        address: u64,
+        before_publish: impl FnOnce(),
+    ) -> Result<u32> {
         if cursors.producer != expected
             || cursors.count != ITEM_RING_ENTRIES
             || expected >= ITEM_RING_ENTRIES
@@ -2777,7 +2787,8 @@ impl Queue {
                         self.pool.cancel_unpublished(target)?; Err(error)
                     })?;
                 self.active.back_mut().ok_or(EIO)?.qos_pending = true;
-                producer = self.graph.append(cursors.producer, config_va.ok_or(EINVAL)?,
+                // `cursors` is this publication's snapshot: no second read of the ring.
+                producer = self.graph.append_to(cursors, cursors.producer, config_va.ok_or(EINVAL)?,
                     || packet.mark_published())
                     .or_else(|error| {
                         self.pool.cancel_unpublished(target)?;
@@ -2878,7 +2889,13 @@ impl Queue {
                 self.graph.record(record_slot, ANNOUNCE, 0x40,
                     crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)?
             } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? };
-            producer = self.graph.append(producer, announce_va, || {})?;
+            producer = match snapshot {
+                // The snapshot's consumer only lags the firmware's, and the free-room check
+                // above covered both M5 items; the producer is the one this publication set.
+                Some(snapshot) => self.graph.append_to(Cursors { producer, ..snapshot }, producer,
+                    announce_va, || {})?,
+                None => self.graph.append(producer, announce_va, || {})?,
+            };
             fence(Ordering::SeqCst);
             if entry_signal && !self.graph.m5 {
                 let predecessor_va =
@@ -2980,6 +2997,11 @@ impl Queue {
     }
 
     fn install_m5(&mut self, packet: &Arc<Packet>, host: &mut impl Host) -> Result {
+        // Installed, configuration released and a kick published: nothing to check, and no
+        // reads of shared memory under the device mutex.
+        if self.m5_install.is_some() && self.ordinal != 0 && self.configuration.is_none() {
+            return Ok(());
+        }
         let (epoch, state) = host.epoch()?;
         if state == 0 { self.retire_configuration(epoch)?; }
         if let Some(end) = self.m5_install {

drivers/gpu/drm/asahi/g17/queue/render.rs

@@ -885,8 +885,17 @@ impl Pair {
             }
             publication::required(cache[stage], config)
         });
-        let cursors = self.memory.graph.cursors()?;
-        if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] {
+        // An idle pair's next command starts from the rings' cursors; behind a command in
+        // flight it starts from that command's windows, and only the producers are checked.
+        let cursors = match self.active.front() {
+            Some(_) => None,
+            None => Some(self.memory.graph.cursors()?),
+        };
+        let producers = match cursors {
+            Some(cursors) => cursors.map(|cursors| cursors.write),
+            None => self.memory.graph.write_cursors()?,
+        };
+        if producers != self.cursors {
             return Err(EBUSY);
         }
         let queues = qids.map(|qid| self.memory.graph.queue_va(qid));
@@ -1022,16 +1031,18 @@ impl Pair {
             )?,
         ];
         let windows = [lanes[0].window, lanes[1].window];
-        let previous = self.active.front().map_or(cursors, |active| {
-            active.ticket.windows.map(|window| {
+        let previous = match (self.active.front(), cursors) {
+            (Some(active), _) => active.ticket.windows.map(|window| {
                 let target = window.target();
                 retirement::Cursors {
                     done: target,
                     read: target,
                     write: target,
                 }
-            })
-        });
+            }),
+            (None, Some(cursors)) => cursors,
+            (None, None) => return Err(EIO),
+        };
         let fragment_slot = WorkSlot::new(
             DataMaster::Fragment,
             queues[1],

drivers/gpu/drm/asahi/g17/queue/render/memory.rs

@@ -283,6 +283,17 @@ impl Graph {
         }
         Ok(values)
     }
+    /// The two rings' producers alone: the `write` of [`Self::cursors`].
+    pub(super) fn write_cursors(&self) -> Result<[u32; 2]> {
+        let mut values = [0; 2];
+        for (stage, value) in values.iter_mut().enumerate() {
+            *value = self
+                .queues
+                .word(POINTERS[stage] + self.pointers.producer)?
+                .load(Ordering::Relaxed);
+        }
+        Ok(values)
+    }
     pub(super) fn claim(&mut self, ordinal: u64) -> Result {
         let owner = &mut self.owners[ordinal as usize % RECORDS];
         if owner.is_some_and(|old| old != ordinal) {

drivers/gpu/drm/asahi/g17/queue/render/observation.rs

@@ -64,18 +64,8 @@ pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result<retireme
         return Err(EIO);
     }
     let cursors = memory.graph.cursors()?;
-    // Sample the private head before the logical shared head, whose membership
-    // may include other pairs. Individual completion does not require it empty.
-    let _first = memory
-        .graph
-        .queues
-        .dword(memory::JOB_LIST)?
-        .load(Ordering::Relaxed);
-    let _last = memory
-        .graph
-        .queues
-        .dword(memory::JOB_LIST + 8)?
-        .load(Ordering::Relaxed);
+    // The logical shared head, whose membership may include other pairs. Individual
+    // completion does not require it empty.
     let head = memory.context.work_head()?;
     Ok(retirement::Snapshot {
         cursors,