diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs
index 5ae76639f6521..e6077624ba4f2 100644
--- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs
+++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs
@@ -1,21 +1,22 @@
// SPDX-License-Identifier: GPL-2.0-only OR MIT
-//! Event-driven render snapshots outside the device mutex. The preparation
-//! epoch and exact pair owner survive every read, cancellation and retry.
+//! Event-driven render snapshots. A pair's completion memory is read and the
+//! pair retired in one device-mutex hold, inside the preparation epoch.
use super::super::DeferredBatch;
-use crate::g17::queue::render::Pair;
use core::sync::atomic::Ordering;
use kernel::{prelude::*, sync::Arc};
impl crate::g17::Firmware {
- fn take_render_observation(&mut self, start: usize) -> Result)>> {
+ /// The first slot from `start` whose pair has news to observe: published work in flight, both
+ /// queues' notifications seen, and an observation the pair accepts.
+ fn next_render_observation(&self, start: usize) -> Result > {
if self.recovery.pending() {
return Ok(None);
}
let state = crate::g17::recovery::Memory::recovery_state(&self.init)?;
for slot in start..self.queues.render.entries.len() {
- let Some(entry) = self.queues.render.entries[slot].as_mut() else {
+ let Some(entry) = self.queues.render.entries[slot].as_ref() else {
continue;
};
if entry.preparing.is_some() || entry.growing {
@@ -37,35 +38,13 @@ impl crate::g17::Firmware {
self.init.stamp(qids[1])?.load(Ordering::Relaxed),
];
core::sync::atomic::fence(Ordering::Acquire);
- let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else {
- continue;
- };
- entry.borrowed_active = pair.published_in_flight();
- entry.borrowed_dependencies = Some(pair.dependency_snapshot());
- entry.borrowed_switch = pair.switch_in_flight();
- entry.preparing = Some(packet);
- return Ok(Some((slot, entry.pair.take().ok_or(EIO)?)));
+ if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() {
+ return Ok(Some(slot));
+ }
}
Ok(None)
}
- fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result {
- let returning = pair.as_ref().ok_or(EINVAL)?;
- let entry = self.queues.render.entry_mut(slot as u8)?;
- if entry.pair.is_some()
- || entry.preparing.is_none()
- || returning.slot() as usize != slot
- || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid)
- {
- return Err(EIO);
- }
- entry.pair = pair.take();
- entry.borrowed_dependencies = None;
- entry.borrowed_switch = None;
- entry.preparing = None;
- Ok(())
- }
-
pub(super) fn settle_render_terminal(
&mut self,
slot: usize,
@@ -122,59 +101,37 @@ impl crate::g17::Shared {
Err(error) if error == EAGAIN || error == ENODEV => break,
result => result?,
};
- let loan = {
- let mut state = self.state.lock();
- let Some(firmware) = (*state).as_deref_mut() else {
- return Ok(());
- };
- if !epoch.is_current() || self.crashed.load(Ordering::Acquire) {
- break;
- }
- firmware.take_render_observation(cursor)?
+ // Selecting, reading and retiring the front share one hold: a
+ // separate hold for the read would let publishers and the
+ // retirement contend twice per observed pair.
+ let mut state = self.state.lock();
+ let Some(firmware) = (*state).as_deref_mut() else {
+ return Ok(());
};
- let Some((slot, pair)) = loan else {
+ if !epoch.is_current()
+ || firmware.recovery.pending()
+ || self.crashed.load(Ordering::Acquire)
+ {
+ break;
+ }
+ let Some(slot) = firmware.next_render_observation(cursor)? else {
break;
};
if slot != cursor {
remaining = crate::hw::t8140::queues::RENDER_DEPTH;
retries = 0;
}
- let observed = pair.observe_front();
- let mut pair = Some(pair);
- let mut state = self.state.lock();
- // Return installed ownership before examining read errors or epoch.
- let restored = (*state)
- .as_deref_mut()
- .ok_or(ENODEV)
- .and_then(|firmware| firmware.return_render_observation(slot, &mut pair));
- if let Err(error) = restored {
- // A failed invariant must not drop firmware-owned storage. Join
- // the other preparers only after releasing our own epoch, and
- // keep this detached owner until both processors have stopped.
- drop(state);
- drop(epoch);
- self.lose_device_quietly();
- self.stop_runtime();
- if let Some(mut pair) = pair {
- pair.stopped(&mut |event| {
- event.finish();
- Ok(())
- })?;
- }
- return Err(error);
- }
- let firmware = (*state).as_deref_mut().ok_or(EIO)?;
+ let observed = firmware.queues.render.entries[slot]
+ .as_ref()
+ .and_then(|entry| entry.pair.as_ref())
+ .ok_or(EIO)?
+ .observe_front();
+ // Publishers waiting for render capacity recheck once this hold ends.
self.note_changed();
if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) {
// Observation and the growth check run on every pass.
self.queue_event_records();
}
- if !epoch.is_current()
- || firmware.recovery.pending()
- || self.crashed.load(Ordering::Acquire)
- {
- break;
- }
let result = (|| {
let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?;
let progressed = match observed {