diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3d1449bee2147..05dbf028cbc81 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -1371,6 +1371,11 @@ impl Lease { } /// Observes all older release records passed by one consumer snapshot. + /// A sent release whose consumption [`Self::observe_releases`] has yet to witness. + pub(crate) fn awaits_release(&self) -> bool { + self.previous_release.is_some() || matches!(self.phase, LeasePhase::ReleaseSent(_)) + } + pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { if let Some(cursor) = self.previous_release { if control_consumed(consumer, producer, cursor)? { diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 5c333e05f8788..3164097665f5c 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -205,6 +205,9 @@ pub(super) struct InitData { compute_scheduler: Option, compute_active: core::cell::Cell, event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], + /// M5: the submission-history geometry (period, slot count) read under the firmware's + /// generation, with that generation. + history_geometry: core::cell::Cell>, /// Completion-ring objects; a family sets only the rings it has. completion_rings: [Option; 4], stamps: [usize; 2], @@ -308,6 +311,7 @@ impl InitData { compute_scheduler: None, compute_active: core::cell::Cell::new(false), event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], + history_geometry: core::cell::Cell::new(None), completion_rings: [None; 4], stamps: [0; 2], render_pages: None, @@ -691,10 +695,37 @@ impl InitData { Ok(next != consumer && next != consumer_b) } + /// M5 compute: the shared ring's cursors when it has room for one record, as + /// [`Self::work_ready`] judges it. The same critical section may publish against them + /// ([`Self::publish_work_at`]): the firmware's consumers only advance, and the producer + /// is the device mutex holder's. + pub(super) fn m5_compute_room(&self, priority: u8) -> Result> { + use super::fw::queue::DataMaster; + if !self.m5 { + return Err(EINVAL); + } + let cursors = self.work_cursors(priority, DataMaster::Compute)?; + Ok((self.outer_capacity(priority, cursors)? >= 1).then_some(cursors)) + } + + /// M5 render: the shared ring's cursors when it has room for a pair, as + /// [`Self::work_ready`] judges it, for the pair's publication in the same hold + /// ([`Self::m5_work_pair`]). + pub(super) fn m5_render_room(&self, priority: u8) -> Result> { + use super::fw::queue::DataMaster; + if !self.m5 { + return Err(EINVAL); + } + let cursors = self.work_cursors(priority, DataMaster::Compute)?; + Ok((self.outer_capacity(priority, cursors)? >= 2).then_some(cursors)) + } + + /// `cursors`: an M5 compute room check's sample ([`Self::m5_compute_room`]), else read here. fn work_publication<'a>( &'a self, priority: u8, record: &'a super::fw::channels::WorkSlot, + cursors: Option<[u32; 3]>, ) -> Result> { use super::fw::{channels, queue::DataMaster}; let engine = match record.data_master { @@ -706,7 +737,11 @@ impl InitData { _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; + let [consumer, consumer_b, producer] = match cursors { + Some(cursors) if self.m5 && engine == DataMaster::Compute => cursors, + Some(_) => return Err(EINVAL), + None => self.work_cursors(priority, engine)?, + }; if self.m5 && engine == DataMaster::Compute && self.outer_capacity(priority, [consumer, consumer_b, producer])? < 1 { return Err(EBUSY); @@ -736,6 +771,21 @@ impl InitData { }) } + /// The host producer of the shared M5 ring alone (the last of [`Self::work_cursors`]). + pub(super) fn work_producer(&self, priority: u8) -> Result { + use super::fw::{channels, queue::DataMaster}; + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let producer = self + .object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER)? + .load(Ordering::Relaxed); + if producer >= channels::WORK_RING_SLOTS { + return Err(EINVAL); + } + Ok(producer) + } + /// Two firmware consumers and the host producer, sampled in that order. pub(super) fn work_cursors( &self, @@ -769,16 +819,30 @@ impl InitData { priority: u8, record: &super::fw::channels::WorkSlot, ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) + self.publish_work_at(priority, record, None) + } + + /// [`Self::publish_work`] against the cursors of a room check made earlier under the same + /// hold of the device mutex ([`Self::m5_compute_room`]), or a fresh read for None. + pub(super) fn publish_work_at( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + cursors: Option<[u32; 3]>, + ) -> Result { + Ok(self.work_publication(priority, record, cursors)?.commit()) } /// Reserve both common-ring slots before touching either host port. The /// returned views retain the graph borrow until the pair is published. + /// `cursors`: a room check's sample from the same hold ([`Self::m5_render_room`]), + /// else read here. pub(super) fn m5_work_pair( &self, priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, + cursors: Option<[u32; 3]>, ) -> Result> { use super::fw::{channels, queue::DataMaster}; if !self.m5 @@ -787,7 +851,10 @@ impl InitData { { return Err(EINVAL); } - let cursors = self.work_cursors(priority, DataMaster::Compute)?; + let cursors = match cursors { + Some(cursors) => cursors, + None => self.work_cursors(priority, DataMaster::Compute)?, + }; if self.outer_capacity(priority, cursors)? < 2 { return Err(EBUSY); } let window = work_pair::Window::new(cursors)?; let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; @@ -829,14 +896,14 @@ impl InitData { return Err(EINVAL); } if self.m5 { - let pair = self.m5_work_pair(priority, fragment, tiling)?; + let pair = self.m5_work_pair(priority, fragment, tiling, None)?; pair.prepare(); commit_accounting()?; publish_late_tiling(); return Ok(if deferred { pair.next() } else { pair.publish() }); } - let fragment = self.work_publication(priority, fragment)?; - let tiling = self.work_publication(priority, tiling)?; + let fragment = self.work_publication(priority, fragment, None)?; + let tiling = self.work_publication(priority, tiling, None)?; fragment.copy_record(); commit_accounting()?; fence(Ordering::SeqCst); @@ -891,16 +958,18 @@ impl InitData { /// Release a retained pair after its control transaction. Revalidate the original producer /// witnesses before either copy; both bodies precede the fragment and tiling publications. + /// `cursors`: M5, as for [`Self::m5_work_pair`]. pub(super) fn publish_retained_work_pair( &self, priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, next: [u32; 2], + cursors: Option<[u32; 3]>, ) -> Result { use super::fw::queue::DataMaster; if self.m5 { - let pair = self.m5_work_pair(priority, fragment, tiling)?; + let pair = self.m5_work_pair(priority, fragment, tiling, cursors)?; if pair.next() != next { return Err(EBUSY); } pair.publish(); return Ok(()); @@ -1936,6 +2005,8 @@ impl super::recovery::Memory for InitData { } fn transition(&self, expected: u32, next: u32) -> Result { + // A restarted firmware may lay its submission history out anew. + self.history_geometry.set(None); let state = self .object(self.cluster)? .word(self.recovery_state_offset())?; diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index ec150b54f1bee..2bc0647da9624 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -55,13 +55,22 @@ impl InitData { for _ in 0..64 { let generation = private.word(base + 0x74c)?.load(Ordering::Acquire); if generation & 1 != 0 { core::hint::spin_loop(); continue; } - let config = private.read_words::<4>(base + 0x73c)?; - fence(Ordering::Acquire); - if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) { - core::hint::spin_loop(); continue; - } - let period = u64::from(config[0]) | u64::from(config[1]) << 32; - let count = u64::from(config[2]) | u64::from(config[3]) << 32; + // The firmware moves the generation around every change of the geometry: while it + // reads as it did, the geometry read under it stands, without four more loads. + let (period, count) = match self.history_geometry.get() { + Some((seen, period, count)) if seen == generation => (period, count), + _ => { + let config = private.read_words::<4>(base + 0x73c)?; + fence(Ordering::Acquire); + if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) { + core::hint::spin_loop(); continue; + } + let period = u64::from(config[0]) | u64::from(config[1]) << 32; + let count = u64::from(config[2]) | u64::from(config[3]) << 32; + self.history_geometry.set(Some((generation, period, count))); + (period, count) + } + }; if period == 0 || count == 0 || count > 256 { return Err(EINVAL); } let (counter, frequency) = crate::g17::qos::clock(); let ticks = crate::g17::qos::time_from_host(counter, frequency)?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4e68d0bd74a47..4ef0a5d86965f 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -798,11 +798,13 @@ impl Firmware { commit_accounting: impl FnOnce() -> Result, publish_inner: impl FnOnce(), deferred: bool, + cursors: Option<[u32; 3]>, ) -> Result<[u32; 2]> { let mut commit_accounting = Some(commit_accounting); let mut publish_inner = Some(publish_inner); + let mut cursors = cursors; for attempt in 0..2 { - let pair = self.init.m5_work_pair(priority, &records[0], &records[1])?; + let pair = self.init.m5_work_pair(priority, &records[0], &records[1], cursors.take())?; pair.prepare(); if let Some(commit) = commit_accounting.take() { commit()?; } match self.bridge.regs.submit_m5_work(submission) { diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 0b9854e84de2c..0fd65708922f7 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -328,6 +328,17 @@ impl KernelObject { pub(crate) fn read_words(&self, offset: usize) -> Result<[u32; N]> { let base = self.aligned::<[u32; N]>(offset)?.cast::(); let mut words = [0u32; N]; + // Most views are uncached: where the copy is 8-byte aligned, load two words at a time. + if N % 2 == 0 && base as usize % 8 == 0 { + let pairs = base.cast::(); + for (i, pair) in words.chunks_exact_mut(2).enumerate() { + // SAFETY: As below; the 8-byte loads cover the same aligned `N` words. + let bytes = unsafe { pairs.add(i).read_volatile() }.to_ne_bytes(); + pair[0] = u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + pair[1] = u32::from_ne_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); + } + return Ok(words); + } for (i, word) in words.iter_mut().enumerate() { // SAFETY: `aligned()` checked that all `N` words are within the object and aligned. *word = unsafe { base.add(i).read_volatile() }; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3e9d2002dafc3..c6a8cc2b46082 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2408,7 +2408,17 @@ impl Graph { Ok(object.gpu_va() + start as u64) } fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result { - let cursors = self.cursors()?; + self.append_to(self.cursors()?, expected, address, before_publish) + } + /// Appends against `cursors`: a fresh read, or an M5 publication's snapshot whose + /// producer the publication advanced itself (see `publish_with_native_waits`). + fn append_to( + &self, + cursors: Cursors, + expected: u32, + address: u64, + before_publish: impl FnOnce(), + ) -> Result { if cursors.producer != expected || cursors.count != ITEM_RING_ENTRIES || expected >= ITEM_RING_ENTRIES @@ -2777,7 +2787,8 @@ impl Queue { self.pool.cancel_unpublished(target)?; Err(error) })?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self.graph.append(cursors.producer, config_va.ok_or(EINVAL)?, + // `cursors` is this publication's snapshot: no second read of the ring. + producer = self.graph.append_to(cursors, cursors.producer, config_va.ok_or(EINVAL)?, || packet.mark_published()) .or_else(|error| { self.pool.cancel_unpublished(target)?; @@ -2878,7 +2889,13 @@ impl Queue { self.graph.record(record_slot, ANNOUNCE, 0x40, crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)? } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? }; - producer = self.graph.append(producer, announce_va, || {})?; + producer = match snapshot { + // The snapshot's consumer only lags the firmware's, and the free-room check + // above covered both M5 items; the producer is the one this publication set. + Some(snapshot) => self.graph.append_to(Cursors { producer, ..snapshot }, producer, + announce_va, || {})?, + None => self.graph.append(producer, announce_va, || {})?, + }; fence(Ordering::SeqCst); if entry_signal && !self.graph.m5 { let predecessor_va = @@ -2980,6 +2997,11 @@ impl Queue { } fn install_m5(&mut self, packet: &Arc, host: &mut impl Host) -> Result { + // Installed, configuration released and a kick published: nothing to check, and no + // reads of shared memory under the device mutex. + if self.m5_install.is_some() && self.ordinal != 0 && self.configuration.is_none() { + return Ok(()); + } let (epoch, state) = host.epoch()?; if state == 0 { self.retire_configuration(epoch)?; } if let Some(end) = self.m5_install { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index ca504ed87db91..2b7ee531560b2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -885,8 +885,17 @@ impl Pair { } publication::required(cache[stage], config) }); - let cursors = self.memory.graph.cursors()?; - if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { + // An idle pair's next command starts from the rings' cursors; behind a command in + // flight it starts from that command's windows, and only the producers are checked. + let cursors = match self.active.front() { + Some(_) => None, + None => Some(self.memory.graph.cursors()?), + }; + let producers = match cursors { + Some(cursors) => cursors.map(|cursors| cursors.write), + None => self.memory.graph.write_cursors()?, + }; + if producers != self.cursors { return Err(EBUSY); } let queues = qids.map(|qid| self.memory.graph.queue_va(qid)); @@ -1022,16 +1031,18 @@ impl Pair { )?, ]; let windows = [lanes[0].window, lanes[1].window]; - let previous = self.active.front().map_or(cursors, |active| { - active.ticket.windows.map(|window| { + let previous = match (self.active.front(), cursors) { + (Some(active), _) => active.ticket.windows.map(|window| { let target = window.target(); retirement::Cursors { done: target, read: target, write: target, } - }) - }); + }), + (None, Some(cursors)) => cursors, + (None, None) => return Err(EIO), + }; let fragment_slot = WorkSlot::new( DataMaster::Fragment, queues[1], diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 65615e1087037..e8881cdc0ad80 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -283,6 +283,17 @@ impl Graph { } Ok(values) } + /// The two rings' producers alone: the `write` of [`Self::cursors`]. + pub(super) fn write_cursors(&self) -> Result<[u32; 2]> { + let mut values = [0; 2]; + for (stage, value) in values.iter_mut().enumerate() { + *value = self + .queues + .word(POINTERS[stage] + self.pointers.producer)? + .load(Ordering::Relaxed); + } + Ok(values) + } pub(super) fn claim(&mut self, ordinal: u64) -> Result { let owner = &mut self.owners[ordinal as usize % RECORDS]; if owner.is_some_and(|old| old != ordinal) { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/observation.rs b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs index 3e853115e4f75..8c7e836cf1c90 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/observation.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs @@ -64,18 +64,8 @@ pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result { regs: &'a crate::regs::G17Resources, port: &'a super::port::Port, outer: Option>, + /// M5: the shared compute ring's room check of this publication, by priority. + outer_cursors: Option<(u8, [u32; 3])>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, generation: u64, @@ -1545,7 +1547,11 @@ impl compute::Host for ComputeHost<'_> { } fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { if self.init.is_m5() { - return self.init.publish_work(priority, slot).map(|_| ()); + // The room check's sample serves the first record only: it moves the producer. + let cursors = self.outer_cursors.take() + .filter(|&(checked, _)| checked == priority) + .map(|(_, cursors)| cursors); + return self.init.publish_work_at(priority, slot, cursors).map(|_| ()); } self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } @@ -1712,12 +1718,13 @@ impl super::Firmware { // reservation; dropping it unused returns the work-ring slot. M5 compute takes its // slot on the work ring M5 render shares, under this mutex (InitData::publish_work): // a full ring is refused here, before a queue install or kick changes any state. + let mut outer_cursors = None; let outer = if self.init.is_m5() { - if !self.init.work_ready(packet.context.policy().priority(), - super::fw::queue::DataMaster::Compute)? - { + let priority = packet.context.policy().priority(); + let Some(cursors) = self.init.m5_compute_room(priority)? else { return Err(EAGAIN); - } + }; + outer_cursors = Some((priority, cursors)); None } else { Some(self.port.reserve_compute(packet.context.policy().priority())?) @@ -1745,6 +1752,7 @@ impl super::Firmware { regs: &self.bridge.regs, port: &self.port, outer, + outer_cursors, effort: &mut self.effort, qids: &mut queues.qids, generation: self.recovery.generation(), diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 2b4f43b5685ad..e72fc921834d9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -17,11 +17,8 @@ impl Firmware { if self.recovery.pending() { return Ok(()); } - let state = recovery::Memory::recovery_state(&self.init)?; - if state != 0 { - return Ok(()); - } - self.queues.replay_consumer_wait = None; + // Runs on every event pass and inline publication: with nothing retained for replay, + // return before reading the recovery state from shared memory. let commands = self .queues .compute @@ -31,9 +28,15 @@ impl Firmware { .map(|queue| queue.replay_commands()) .sum::(); if commands == 0 { + self.queues.replay_consumer_wait = None; self.queues.replay_outer_wait = None; return Ok(()); } + let state = recovery::Memory::recovery_state(&self.init)?; + if state != 0 { + return Ok(()); + } + self.queues.replay_consumer_wait = None; deferred.reserve_replays(commands)?; let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 5114465145081..10ba2927f7fbe 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -338,8 +338,10 @@ impl Firmware { for &role in self.roles() { self.rings.drain_trace(dev, &self.init, role); } + // Each drain is bracketed by power observations; consecutive roles share the one + // between them (each reads the GPU's power register). + self.observe_statistics_power(); for &role in self.roles() { - self.observe_statistics_power(); self.rings.drain_statistics(&self.primary.state.shared.dev, &self.init, role); self.observe_statistics_power(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4b62d96d7cd3c..2a3c78e6f32c5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -526,8 +526,36 @@ struct Host<'a> { registration_powered: bool, /// An M5 pair's installation, submitted with its first outer publication. m5_registration: Option, + /// M5: the shared ring's room check of this publication, by priority. Its consumers stay + /// conservative for the whole hold (the firmware only advances them); its producer + /// serves the first pair written to the ring only. + outer_cursors: Option<(u8, [u32; 3])>, + outer_producer_used: bool, } impl Host<'_> { + /// The room check's cursors for the first pair this publication writes to the ring. + fn room_cursors(&mut self, priority: u8) -> Option<[u32; 3]> { + if core::mem::replace(&mut self.outer_producer_used, true) { + return None; + } + self.outer_cursors + .filter(|&(checked, _)| checked == priority) + .map(|(_, cursors)| cursors) + } + + /// The room check's consumers with the producer read now: a retained pair revalidates the + /// producer it was prepared against. + fn retained_cursors(&mut self, priority: u8) -> Result> { + self.outer_producer_used = true; + let Some((checked, [consumer, consumer_b, _])) = self.outer_cursors else { + return Ok(None); + }; + if checked != priority { + return Ok(None); + } + Ok(Some([consumer, consumer_b, self.firmware.init.work_producer(priority)?])) + } + fn release_registration(&mut self) { if !self.registration_powered && !self.firmware.registration_power_release_pending { return; @@ -634,6 +662,7 @@ impl crate::g17::queue::render::Host for Host<'_> { tiling.qid, )?, }; + let cursors = self.room_cursors(priority); return self.firmware.submit_m5_render_pair( priority, &[copy_slot(fragment), copy_slot(tiling)], @@ -641,6 +670,7 @@ impl crate::g17::queue::render::Host for Host<'_> { commit, late_tiling, true, + cursors, ); } self.firmware @@ -654,7 +684,8 @@ impl crate::g17::queue::render::Host for Host<'_> { tiling: &crate::g17::fw::channels::WorkSlot, prepare: impl FnOnce(), ) -> Result { - let pair = self.firmware.init.m5_work_pair(priority, fragment, tiling)?; + let cursors = self.room_cursors(priority); + let pair = self.firmware.init.m5_work_pair(priority, fragment, tiling, cursors)?; pair.prepare(); prepare(); pair.publish(); @@ -677,10 +708,11 @@ impl crate::g17::queue::render::Host for Host<'_> { tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], ) -> Result { - let result = self - .firmware - .init - .publish_retained_work_pair(priority, fragment, tiling, next); + let result = self.retained_cursors(priority).and_then(|cursors| { + self.firmware + .init + .publish_retained_work_pair(priority, fragment, tiling, next, cursors) + }); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } @@ -762,6 +794,17 @@ impl crate::g17::Firmware { &mut self, slot: usize, operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, + ) -> Result { + self.with_render_at(slot, None, operation) + } + + /// [`Self::with_render`] for a publication whose shared-ring room check, made in this + /// hold, sampled `outer_cursors`. + fn with_render_at( + &mut self, + slot: usize, + outer_cursors: Option<(u8, [u32; 3])>, + operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, ) -> Result { let mut pair = self .queues @@ -775,6 +818,8 @@ impl crate::g17::Firmware { firmware: self, registration_powered: false, m5_registration: None, + outer_cursors, + outer_producer_used: false, }; let result = operation(&mut pair, &mut host); host.release_registration(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index ab94764580521..ba287c13456cc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -219,6 +219,11 @@ impl Pools { } pub(super) fn observe(&mut self, init: &InitData) -> Result { + // Every render publication observes: with no release in flight there is nothing to + // witness, and no reason to read the control ring's cursors from shared memory. + if !self.leases[FIRST_RENDER_POOL..].iter().flatten().any(|lease| lease.awaits_release()) { + return Ok(()); + } fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 67488b7299bbc..a37ad6b2c8ab8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -33,6 +33,15 @@ impl super::RenderState { .any(|pair| pair.owner() == Some(owner) && pair.replay_through(through)) } + /// Whether any pair holds a group parked for replay. + pub(in crate::g17) fn replay_parked(&self) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.replay_front().is_some()) + } + pub(in crate::g17) fn replay_polling(&self) -> bool { self.entries .iter() @@ -44,9 +53,12 @@ impl super::RenderState { impl Firmware { pub(in crate::g17) fn replay_render(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&self.init)? != 0 - { + // Runs on every event pass: with no group parked for replay, return before reading the + // recovery state from shared memory. + if self.recovery.pending() || !self.queues.render.replay_parked() { + return Ok(()); + } + if crate::g17::recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } for slot in 0..self.queues.render.entries.len() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 431dbbd748ecf..d35c2cdd66420 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -749,13 +749,23 @@ impl crate::g17::Firmware { let port = self.port.clone(); let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. + // either inner queue becomes visible. On M5 both are the shared compute ring: one + // check covers the pair, and the pair is written against its cursors (this hold + // is the ring's only producer; the firmware's consumers only advance). let priority = packet.context.policy().priority(); - if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? - || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? - { - return Err(EBUSY); - } + let outer_cursors = if self.init.is_m5() { + let Some(cursors) = self.init.m5_render_room(priority)? else { + return Err(EBUSY); + }; + Some((priority, cursors)) + } else { + if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? + { + return Err(EBUSY); + } + None + }; if let Some(footprint) = packet.footprint.as_ref() { dependencies.hazard_count = self.queues.admission.publication_barriers( owner, @@ -796,7 +806,7 @@ impl crate::g17::Firmware { buffer_token, buffer_context: buffer_context.map(|(context, _)| context), }; - let result = self.with_render(usize::from(slot), |pair, host| { + let result = self.with_render_at(usize::from(slot), outer_cursors, |pair, host| { pair.publish( packet.clone(), &dependencies,