drm/asahi: G17 stability fixes from linux-aurora PR #151 (7.3 port)

drivers/base/base.h

@@ -222,6 +222,8 @@ typedef void (*dr_node_free_t)(struct devres_node *node);
 
 struct devres_node {
 	struct list_head		entry;
+	/* Device-list membership, cleared under devres_lock before detachment. */
+	struct device			*owner;
 	dr_node_release_t		release;
 	dr_node_free_t			free_node;
 	const char			*name;

drivers/base/devres.c

@@ -41,6 +41,7 @@ void devres_node_init(struct devres_node *node,
 		      dr_node_free_t free_node)
 {
 	INIT_LIST_HEAD(&node->entry);
+	WRITE_ONCE(node->owner, NULL);
 	node->release = release;
 	node->free_node = free_node;
 }
@@ -156,6 +157,7 @@ static void add_dr(struct device *dev, struct devres_node *node)
 	devres_log(dev, node, "ADD");
 	BUG_ON(!list_empty(&node->entry));
 	list_add_tail(&node->entry, &dev->devres_head);
+	WRITE_ONCE(node->owner, dev);
 }
 
 static void replace_dr(struct device *dev,
@@ -164,6 +166,8 @@ static void replace_dr(struct device *dev,
 	devres_log(dev, old, "REPLACE");
 	BUG_ON(!list_empty(&new->entry));
 	list_replace(&old->entry, &new->entry);
+	WRITE_ONCE(old->owner, NULL);
+	WRITE_ONCE(new->owner, dev);
 }
 
 /**
@@ -367,18 +371,14 @@ EXPORT_SYMBOL_GPL(devres_get);
 
 bool devres_node_remove(struct device *dev, struct devres_node *node)
 {
-	struct devres_node *__node;
-
 	guard(spinlock_irqsave)(&dev->devres_lock);
-	list_for_each_entry_reverse(__node, &dev->devres_head, entry) {
-		if (__node == node) {
-			list_del_init(&node->entry);
-			devres_log(dev, node, "REM");
-			return true;
-		}
-	}
-
-	return false;
+	/* A release-all/group pass may own a detached node on its private list. */
+	if (READ_ONCE(node->owner) != dev)
+		return false;
+	list_del_init(&node->entry);
+	WRITE_ONCE(node->owner, NULL);
+	devres_log(dev, node, "REM");
+	return true;
 }
 
 /**
@@ -405,6 +405,7 @@ void *devres_remove(struct device *dev, dr_release_t release,
 	dr = find_dr(dev, release, match, match_data);
 	if (dr) {
 		list_del_init(&dr->node.entry);
+		WRITE_ONCE(dr->node.owner, NULL);
 		devres_log(dev, &dr->node, "REM");
 		return dr->data;
 	}
@@ -500,6 +501,7 @@ static int remove_nodes(struct device *dev,
 			if (&node->entry == first)
 				first = first->next;
 			list_move_tail(&node->entry, todo);
+			WRITE_ONCE(node->owner, NULL);
 			cnt++;
 		}
 	}
@@ -530,7 +532,9 @@ static int remove_nodes(struct device *dev,
 			 * nodes are always before both.
 			 */
 			list_move_tail(&grp->node[0].entry, todo);
+			WRITE_ONCE(grp->node[0].owner, NULL);
 			list_del_init(&grp->node[1].entry);
+			WRITE_ONCE(grp->node[1].owner, NULL);
 		}
 	}
 
@@ -685,7 +689,9 @@ void devres_remove_group(struct device *dev, void *id)
 	grp = find_group(dev, id);
 	if (grp) {
 		list_del_init(&grp->node[0].entry);
+		WRITE_ONCE(grp->node[0].owner, NULL);
 		list_del_init(&grp->node[1].entry);
+		WRITE_ONCE(grp->node[1].owner, NULL);
 		devres_log(dev, &grp->node[0], "REM");
 	} else
 		WARN_ON(1);
@@ -849,6 +855,7 @@ static struct devres_action *remove_action(struct device *dev,
 		return ERR_PTR(-ENOENT);
 
 	list_del_init(&dr->node.entry);
+	WRITE_ONCE(dr->node.owner, NULL);
 	devres_log(dev, &dr->node, "REM");
 
 	return dr;

drivers/gpu/drm/asahi/g17/capacity.rs

@@ -24,9 +24,10 @@ pub(super) enum Site {
     LowerVa,
     UpperVa,
     AliasVa,
+    ExecutionContexts,
 }
 
-const SITE_COUNT: usize = Site::AliasVa as usize + 1;
+const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1;
 static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT];
 
 /// First and power-of-two refusals per site keep a pressure storm bounded in
@@ -78,7 +79,7 @@ impl kernel::debugfs::Writer for View {
             Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues,
             Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings,
             Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds,
-            Site::LowerVa, Site::UpperVa, Site::AliasVa,
+            Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts,
         ].iter().zip(REFUSALS.iter()) {
             writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?;
         }

drivers/gpu/drm/asahi/g17/completion.rs

@@ -144,7 +144,8 @@ impl HostOutput {
     }
 }
 
-/// Firmware writes two aligned words in a fresh write-combined page per compute command.
+/// Firmware writes two aligned words in a write-combined page of its own per
+/// compute command.
 struct ComputeTimestamps {
     // Remove the GPU alias before releasing its backing.
     mapping: mmu::KernelMapping,
@@ -166,14 +167,89 @@ impl ComputeTimestamps {
     }
 }
 
+/// Compute timestamp pages of one context, kept mapped for its next commands.
+///
+/// A fresh page costs a GEM object, its write-combined kernel mapping and a
+/// GPU mapping on the submission path, and their teardown when the command's
+/// completion is dropped. A page comes back here only from that drop, which
+/// follows exact retirement or a kill witness, so the firmware no longer
+/// writes it; both words are cleared before the page is handed out again.
+#[pin_data]
+pub(crate) struct ComputeTimestampCache {
+    #[pin]
+    pages: Mutex<KVec<ComputeTimestamps>>,
+}
+
+impl ComputeTimestampCache {
+    /// Pages retained per context; deeper pipelines allocate the rest.
+    const RETAINED: usize = 8;
+
+    pub(crate) fn new() -> Result<Arc<Self>> {
+        let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?;
+        Arc::pin_init(
+            pin_init!(Self {
+                pages <- new_mutex!(pages, "G17 compute timestamp pages"),
+            }),
+            GFP_KERNEL,
+        )
+    }
+
+    fn take(self: &Arc<Self>, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result<PooledTimestamps> {
+        let reused = self.pages.lock().pop();
+        let page = match reused {
+            Some(page) => {
+                page.object.dword(0)?.store(0, Ordering::Relaxed);
+                page.object.dword(8)?.store(0, Ordering::Relaxed);
+                barrier(Ordering::SeqCst);
+                page
+            }
+            None => ComputeTimestamps::new(dev, vm)?,
+        };
+        Ok(PooledTimestamps {
+            page: Some(page),
+            cache: self.clone(),
+        })
+    }
+
+    fn put(&self, page: ComputeTimestamps) {
+        let rejected = {
+            let mut pages = self.pages.lock();
+            pages.push_within_capacity(page).err()
+        };
+        // Teardown of a page beyond the retained count runs unlocked.
+        drop(rejected);
+    }
+}
+
+/// A compute timestamp page lent to one command, returned to its context's
+/// cache when the command's completion is dropped.
+struct PooledTimestamps {
+    page: Option<ComputeTimestamps>,
+    cache: Arc<ComputeTimestampCache>,
+}
+
+impl PooledTimestamps {
+    fn page(&self) -> Result<&ComputeTimestamps> {
+        self.page.as_ref().ok_or(EINVAL)
+    }
+}
+
+impl Drop for PooledTimestamps {
+    fn drop(&mut self) {
+        if let Some(page) = self.page.take() {
+            self.cache.put(page);
+        }
+    }
+}
+
 #[pin_data]
 pub(crate) struct Completion {
     status: Arc<VmStatus>,
     output: HostOutput,
     // Present only when this command's public result may wait for a render
     // predecessor's host writes. DRM credits must follow the terminal result.
     terminal: Option<UserFence<CompletionFence>>,
-    compute: Option<ComputeTimestamps>,
+    compute: Option<PooledTimestamps>,
     feed: Option<Arc<super::feed::Feed>>,
     #[pin]
     vm_job: Mutex<Option<mmu::VmJobGuard>>,
@@ -220,7 +296,7 @@ impl Completion {
             HostOutput::Immediate(output)
         };
         let compute = if compute {
-            Some(ComputeTimestamps::new(dev, context.vm())?)
+            Some(context.compute_timestamps().take(dev, context.vm())?)
         } else {
             None
         };
@@ -282,14 +358,14 @@ impl Completion {
     }
 
     pub(crate) fn compute_timestamp_va(&self) -> Result<u64> {
-        Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova())
+        Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova())
     }
 
     /// The caller has proved that firmware retired this command, or has
     /// received its scheduler kill acknowledgement with recovery closed.
     /// These are the command's public timestamp words, not queue-private state.
     pub(crate) fn compute_words(&self) -> Result<[u64; 2]> {
-        let object = &self.compute.as_ref().ok_or(EINVAL)?.object;
+        let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object;
         let words = [
             object.dword(0)?.load(Ordering::Relaxed),
             object.dword(8)?.load(Ordering::Relaxed),
@@ -339,7 +415,7 @@ impl Completion {
         if self.signalled.load(Ordering::Acquire) {
             return Err(ECANCELED);
         }
-        let object = &self.compute.as_ref().ok_or(EINVAL)?.object;
+        let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object;
         let start = object.dword(0)?;
         let end = object.dword(8)?;
         start.store(0, Ordering::Relaxed);
@@ -405,6 +481,12 @@ impl Completion {
         Ok(())
     }
 
+    /// An ioctl publication that met contention returns its lease before the
+    /// scheduler publishes the command. Nothing referred to it yet.
+    pub(crate) fn detach_render_lease(&self) -> Option<super::queue::admission::RenderLease> {
+        self.render.lock().take()
+    }
+
     pub(crate) fn render_slot(&self) -> Option<u8> {
         self.render
             .lock()

drivers/gpu/drm/asahi/g17/context.rs

@@ -121,6 +121,8 @@ pub(crate) struct Context {
     work_base: u64,
     render: Arc<Mutex<Option<Arc<RenderPool>>>>,
     render_blocks: AtomicUsize,
+    /// Timestamp pages of retired compute commands, mapped in this context's VM.
+    compute_timestamps: Arc<super::completion::ComputeTimestampCache>,
     /// Fixed for the context's life. Physical queues bound to it carry this
     /// profile in every record, ring, doorbell and registration.
     policy: Policy,
@@ -194,12 +196,16 @@ impl Context {
             work_base,
             render,
             render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS),
+            compute_timestamps: super::completion::ComputeTimestampCache::new()?,
             policy,
         })
     }
     pub(crate) fn vm(&self) -> &Vm {
         self.execution.vm()
     }
+    pub(crate) fn compute_timestamps(&self) -> &Arc<super::completion::ComputeTimestampCache> {
+        &self.compute_timestamps
+    }
     pub(crate) fn status(&self) -> &Arc<VmStatus> {
         &self.status
     }