M3 Pro GPU crash fix: preserve new mappings during deferred GEM close

drivers/gpu/drm/asahi/mmu.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! GPU UAT (MMU) management
@@ -312,6 +312,7 @@
     va_range: Range<u64>,
     page_table: UatPageTable,
     mapping_epoch: Option<Arc<MappingEpoch>>,
+    bind_sequence: Arc<AtomicU64>,
     epoch: Option<u64>,
     mm: mm::Allocator<(), KernelMappingInner>,
     uat_inner: Arc<UatInner>,
@@ -368,6 +369,11 @@
 /// Data associated with a VM <=> BO pairing
 #[pin_data]
 struct VmBoData {
+    /// Highest sequence assigned to a GPUVA linked to this VM/BO pair.
+    ///
+    /// Map steps update this monotonically under the VM reservation before
+    /// linking the GPUVA. Remap pieces keep their original sequence.
+    newest_bind: AtomicU64,
     #[pin]
     segments: Mutex<Option<KVVec<(DmaAddress, ResourceSize)>>>,
 }
@@ -410,6 +416,9 @@
 
     /// Preallocated resources that can be used when executing the request.
     resources: &'ctx mut VmOpResources,
+
+    /// Sequence assigned by the successful map for its related metadata.
+    bind_sequence: u64,
 }
 
 impl<'ctx> StepContext<'ctx> {
@@ -422,6 +431,7 @@
             dev,
             prot,
             resources,
+            bind_sequence: 0,
         })
     }
 
@@ -452,6 +462,7 @@
 #[derive(Default, Clone)]
 struct VmGpuVa {
     prot: Option<Prot>,
+    bind_sequence: u64,
 }
 
 impl gpuvm::DriverGpuVm for VmInner {
@@ -470,6 +481,11 @@
         op: OpMap<'op, Self>,
         ctx: &mut Self::SmContext<'ctx>,
     ) -> Result<OpMapped<'op, Self>, Error> {
+        let bind_sequence = self.bind_sequence.fetch_update(
+            Ordering::AcqRel,
+            Ordering::Acquire,
+            |value| value.checked_add(1),
+        ).map_err(|_| EOVERFLOW)? + 1;
         let _mutation = self.mapping_mutation();
         let (map_addr, map_length) = (op.addr(), op.length() as usize);
         let mut iova = op.addr();
@@ -547,12 +563,18 @@
                 return Err(error);
             }
         };
+        op.vm_bo()
+            .data()
+            .newest_bind
+            .fetch_max(bind_sequence, Ordering::Relaxed);
         let op = op.insert(
             gpuva,
             VmGpuVa {
                 prot: Some(ctx.prot),
+                bind_sequence,
             },
         );
+        ctx.bind_sequence = bind_sequence;
 
         self.tlbi_contexts_naming_root(map_addr, map_length);
         Ok(op)
@@ -578,6 +600,7 @@
         ctx: &mut Self::SmContext<'ctx>,
     ) -> Result<OpRemapped<'op, Self>, Error> {
         let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?;
+        let orig_bind_sequence = op.va_to_unmap().data().bind_sequence;
         let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED);
 
         // Only unmap the hole between prev/next, if they exist
@@ -642,9 +665,11 @@
             [prev_gpuva, next_gpuva],
             VmGpuVa {
                 prot: Some(orig_prot),
+                bind_sequence: orig_bind_sequence,
             },
             VmGpuVa {
                 prot: Some(orig_prot),
+                bind_sequence: orig_bind_sequence,
             },
         );
 
@@ -882,6 +907,7 @@
     dev: ARef<device::Device>,
     id: u64,
     mapping_epoch: Option<Arc<MappingEpoch>>,
+    bind_sequence: Arc<AtomicU64>,
     lifetime: Option<Arc<lifetime::VmLifetime>>,
     residency: Option<Arc<residency::Gate>>,
     context_bindings: Option<Arc<context::ContextBindings>>,
@@ -1704,6 +1730,7 @@
         } else {
             None
         };
+        let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?;
         let unique_gpuvm = gpuvm::GpuVm::new::<Error, _>(
             c"Asahi::GpuVm",
             &dev,
@@ -1716,6 +1743,7 @@
                 va_range: va_range.clone(),
                 page_table,
                 mapping_epoch: mapping_epoch.clone(),
+                bind_sequence: bind_sequence.clone(),
                 epoch: Some(0),
                 mm,
                 uat_inner,
@@ -1730,6 +1758,7 @@
             id,
             gpuvm,
             mapping_epoch,
+            bind_sequence,
             lifetime: None,
             residency: None,
             context_bindings: None,
@@ -1894,6 +1923,10 @@
         self.mapping_epoch.as_ref()?.snapshot()
     }
 
+    fn current_bind_sequence(&self) -> u64 {
+        self.bind_sequence.load(Ordering::Acquire)
+    }
+
     /// Get the translation table base for this Vm
     fn ttb(&self) -> u64 {
         self.binding.lock().ttb
@@ -1950,7 +1983,10 @@
         }
         let vm_bo = self
             .gpuvm
-            .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?;
+            .obtain(gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
 
         let size = object_range.range();
         let mut vm_exec = unsafe {
@@ -2031,7 +2067,10 @@
         }
         let vm_bo = self
             .gpuvm
-            .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?;
+            .obtain(&gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
 
         // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe
         let mut vm_exec = unsafe {
@@ -2112,7 +2151,10 @@
         }
         let vm_bo = self
             .gpuvm
-            .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?;
+            .obtain(gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
 
         let mut resources = VmOpResources {
             preallocated_gpuvas: [
@@ -2338,6 +2380,65 @@
         Ok(())
     }
 
+    fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result {
+        let _residency = self.enter_retirement_residency();
+        // Removing whole mappings only does unmaps, so no preallocated VAs
+        // are needed for the indexed path.
+        let mut resources = VmOpResources {
+            preallocated_gpuvas: [None, None, None],
+        };
+        let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?;
+
+        // The BO's newest sequence is checked under the same lock as binds. If
+        // it is not newer than the close, every mapping belongs to that close
+        // and the per-object list is safe and substantially cheaper to walk.
+        let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } {
+            Err(error) if error == ENOMEM => {
+                return self.drop_mappings_filtered(gem, sequence);
+            }
+            result => result?,
+        };
+        let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else {
+            return Ok(());
+        };
+        if bo.data().newest_bind.load(Ordering::Relaxed) > sequence {
+            drop(vm_exec);
+            drop(bo);
+            return self.drop_mappings_filtered(gem, sequence);
+        }
+
+        mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n");
+        let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo);
+        if result.is_ok() {
+            self.untrack_context_object_before(gem, sequence);
+            self.untrack_shared_object_before(gem, sequence);
+        }
+        drop(vm_exec);
+        drop(bo);
+        if result == Err(ENOMEM) {
+            return self.drop_mappings_filtered(gem, sequence);
+        }
+        result
+    }
+
+    /// A newer binding shares the BO list, so select the closed generation by
+    /// walking the VM without allocating. This is also the ENOMEM fallback.
+    fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result {
+        let mut inner = self.gpuvm.lock_inner();
+        let result = inner.unmap_object_where(
+            gem,
+            |va| va.data().bind_sequence <= sequence,
+            VmInner::unmap_gpuva,
+        );
+        if result.is_ok() {
+            self.untrack_context_object_before(gem, sequence);
+            self.untrack_shared_object_before(gem, sequence);
+        }
+        drop(inner);
+        self.bo_deferred_cleanup();
+        result
+    }
+
     /// Preserve the indexed fast path normally; memory pressure must not lose
     /// accepted object cleanup just because an exec or operation list failed.
     fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result {

drivers/gpu/drm/asahi/mmu/bind.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Prepared userspace mapping transactions. Host callback resources are reserved for the
@@ -83,7 +83,10 @@
         crate::gem::held_sg_table(gem, dev)?;
         let vm_bo = self
             .gpuvm
-            .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?;
+            .obtain(gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
         {
             let mut segments = vm_bo.data().segments.lock();
             if segments.is_none() {
@@ -156,13 +159,18 @@
                                 flags,
                             });
                             let retired = result.is_ok().then(|| {
-                                let retired = self.commit_context_binding(&map.gem, &map.context);
+                                let retired = self.commit_context_binding(
+                                    &map.gem,
+                                    &map.context,
+                                    ctx.bind_sequence,
+                                );
                                 self.track_shared_binding(
                                     &map.gem,
                                     map.addr,
                                     map.size,
                                     map.offset,
                                     map.single_page,
+                                    ctx.bind_sequence,
                                 );
                                 retired
                             });

drivers/gpu/drm/asahi/mmu/context.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Tracks canonical context resources and retains their fixed low presentation.
@@ -14,6 +14,7 @@
 struct Binding {
     gem: ARef<gem::Object>,
     offset: usize,
+    bind_sequence: u64,
 }
 struct Bindings {
     generation: u64,
@@ -115,6 +116,7 @@
         &self,
         gem: &ARef<gem::Object>,
         update: &PreparedBinding,
+        bind_sequence: u64,
     ) -> RetiredBindings {
         let mut retired = RetiredBindings::new();
         if update.touched == 0 {
@@ -136,6 +138,7 @@
                 *slot = Some(Binding {
                     gem: gem.clone(),
                     offset,
+                    bind_sequence,
                 });
             }
         }
@@ -173,6 +176,10 @@
     }
 
     pub(crate) fn untrack_context_object(&self, gem: &gem::Object) {
+        self.untrack_context_object_before(gem, u64::MAX);
+    }
+
+    pub(super) fn untrack_context_object_before(&self, gem: &gem::Object, sequence: u64) {
         let Some(context) = self.context_bindings.as_ref() else {
             return;
         };
@@ -181,7 +188,8 @@
         for slot in &mut bindings.slots {
             if slot
                 .as_ref()
-                .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem))
+                .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)
+                    && binding.bind_sequence <= sequence)
             {
                 *slot = None;
                 touched = true;

drivers/gpu/drm/asahi/mmu/lifetime.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Keeps user mappings alive until accepted work has relinquished firmware ownership.
@@ -47,6 +47,12 @@
     }
 }
 
+struct ClosedObject {
+    gem: ARef<gem::Object>,
+    /// Highest binding sequence covered by the close or merged closes.
+    bind_sequence: u64,
+}
+
 struct State {
     active: usize,
     commits: usize,
@@ -57,7 +63,7 @@
     cleanup_failed: bool,
     close_ranges: Option<(Range<u64>, Range<u64>)>,
     unmaps: KVec<super::bind::PreparedUserUnmap>,
-    objects: KVec<ARef<gem::Object>>,
+    objects: KVec<ClosedObject>,
 }
 
 #[pin_data]
@@ -545,18 +551,22 @@
             return self.drop_mappings_now(gem);
         };
         let mut state = lifetime.state.lock();
-        if !state
-            .objects
-            .iter()
-            .any(|object| core::ptr::eq(&**object, gem))
-        {
-            state.objects.push(gem.into(), GFP_KERNEL)?;
+        let bind_sequence = self.current_bind_sequence();
+        if let Some(object) = state.objects.iter_mut().find(|object| {
+            core::ptr::eq(&*object.gem, gem)
+        }) {
+            object.bind_sequence = object.bind_sequence.max(bind_sequence);
+        } else {
+            state.objects.push(ClosedObject {
+                gem: gem.into(),
+                bind_sequence,
+            }, GFP_KERNEL)?;
         }
         if state.active != 0 || state.draining {
             // Stop new jobs from selecting this accepted cleanup candidate.
             // The queued object retains its GEM while the candidate is removed.
-            // Actual unmap repeats this under exec to order any intervening bind.
-            self.untrack_context_object(gem);
+            // Preserve a context view installed by a later bind.
+            self.untrack_context_object_before(gem, bind_sequence);
             return Ok(());
         }
         state.draining = true;
@@ -641,7 +651,7 @@
                 }
             }
             for object in objects {
-                if let Err(error) = self.drop_mappings_now(&object) {
+                if let Err(error) = self.drop_mappings_before(&object.gem, object.bind_sequence) {
                     failure.get_or_insert(error);
                     pr_err!("MMU: deferred object unmap failed\n");
                 }