omacom sync: M3 Air GPU lane (GPUVM deferred close fixes; T8122 runtime held)

drivers/gpu/drm/asahi/mmu.rs

@@ -366,6 +366,7 @@ struct VmInner {
     va_range: Range<u64>,
     page_table: UatPageTable,
     mapping_epoch: Option<Arc<MappingEpoch>>,
+    bind_sequence: Arc<AtomicU64>,
     epoch: Option<u64>,
     mm: mm::Allocator<(), KernelMappingInner>,
     uat_inner: Arc<UatInner>,
@@ -424,6 +425,11 @@ struct VmBinding {
 /// Data associated with a VM <=> BO pairing
 #[pin_data]
 struct VmBoData {
+    /// Highest sequence assigned to a GPUVA linked to this VM/BO pair.
+    ///
+    /// Map steps update this monotonically under the VM reservation before
+    /// linking the GPUVA. Remap pieces keep their original sequence.
+    newest_bind: AtomicU64,
     #[pin]
     segments: Mutex<Option<SgSegments>>,
 }
@@ -479,6 +485,9 @@ struct StepContext<'ctx> {
 
     /// Preallocated resources that can be used when executing the request.
     resources: &'ctx mut VmOpResources,
+
+    /// Sequence assigned by the successful map for its related metadata.
+    bind_sequence: u64,
 }
 
 impl<'ctx> StepContext<'ctx> {
@@ -491,6 +500,7 @@ impl<'ctx> StepContext<'ctx> {
             dev,
             prot,
             resources,
+            bind_sequence: 0,
         })
     }
 
@@ -521,6 +531,7 @@ impl<'ctx> StepContext<'ctx> {
 #[derive(Default, Clone)]
 struct VmGpuVa {
     prot: Option<Prot>,
+    bind_sequence: u64,
 }
 
 impl gpuvm::DriverGpuVm for VmInner {
@@ -539,6 +550,11 @@ impl gpuvm::DriverGpuVm for VmInner {
         op: OpMap<'op, Self>,
         ctx: &mut Self::SmContext<'ctx>,
     ) -> Result<OpMapped<'op, Self>, Error> {
+        let bind_sequence = self.bind_sequence.fetch_update(
+            Ordering::AcqRel,
+            Ordering::Acquire,
+            |value| value.checked_add(1),
+        ).map_err(|_| EOVERFLOW)? + 1;
         let _mutation = self.mapping_mutation();
         let (map_addr, map_length) = (op.addr(), op.length() as usize);
         let mut iova = op.addr();
@@ -611,12 +627,18 @@ impl gpuvm::DriverGpuVm for VmInner {
                 return Err(error);
             }
         };
+        op.vm_bo()
+            .data()
+            .newest_bind
+            .fetch_max(bind_sequence, Ordering::Relaxed);
         let op = op.insert(
             gpuva,
             VmGpuVa {
                 prot: Some(ctx.prot),
+                bind_sequence,
             },
         );
+        ctx.bind_sequence = bind_sequence;
 
         self.tlbi_contexts_naming_root(map_addr, map_length);
         Ok(op)
@@ -642,6 +664,7 @@ impl gpuvm::DriverGpuVm for VmInner {
         ctx: &mut Self::SmContext<'ctx>,
     ) -> Result<OpRemapped<'op, Self>, Error> {
         let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?;
+        let orig_bind_sequence = op.va_to_unmap().data().bind_sequence;
         let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED);
 
         // Only unmap the hole between prev/next, if they exist
@@ -707,9 +730,11 @@ impl gpuvm::DriverGpuVm for VmInner {
             [prev_gpuva, next_gpuva],
             VmGpuVa {
                 prot: Some(orig_prot),
+                bind_sequence: orig_bind_sequence,
             },
             VmGpuVa {
                 prot: Some(orig_prot),
+                bind_sequence: orig_bind_sequence,
             },
         );
 
@@ -994,6 +1019,7 @@ pub(crate) struct Vm {
     dev: ARef<device::Device>,
     id: u64,
     mapping_epoch: Option<Arc<MappingEpoch>>,
+    bind_sequence: Arc<AtomicU64>,
     lifetime: Option<Arc<lifetime::VmLifetime>>,
     residency: Option<Arc<residency::Gate>>,
     context_bindings: Option<Arc<context::ContextBindings>>,
@@ -2112,6 +2138,7 @@ impl Vm {
         } else {
             None
         };
+        let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?;
         let unique_gpuvm = gpuvm::GpuVm::new::<Error, _>(
             c"Asahi::GpuVm",
             &dev,
@@ -2124,6 +2151,7 @@ impl Vm {
                 va_range: va_range.clone(),
                 page_table,
                 mapping_epoch: mapping_epoch.clone(),
+                bind_sequence: bind_sequence.clone(),
                 epoch: Some(0),
                 mm,
                 uat_inner,
@@ -2140,6 +2168,7 @@ impl Vm {
             id,
             gpuvm,
             mapping_epoch,
+            bind_sequence,
             lifetime: None,
             residency: None,
             context_bindings: None,
@@ -2304,6 +2333,10 @@ impl Vm {
         self.mapping_epoch.as_ref()?.snapshot()
     }
 
+    fn current_bind_sequence(&self) -> u64 {
+        self.bind_sequence.load(Ordering::Acquire)
+    }
+
     /// Get the translation table base for this Vm
     fn ttb(&self) -> u64 {
         self.binding.lock().ttb
@@ -2404,7 +2437,8 @@ impl Vm {
             let bo = self.gpuvm.obtain(
                 gem,
                 pin_init!(VmBoData {
-                    segments <- new_mutex!(None)
+                    newest_bind: AtomicU64::new(0),
+                    segments <- new_mutex!(None),
                 }),
             )?;
             prepare_sgt_segments(dev, gem, &bo, &lease)?;
@@ -2492,7 +2526,8 @@ impl Vm {
             let bo = self.gpuvm.obtain(
                 gem,
                 pin_init!(VmBoData {
-                    segments <- new_mutex!(None)
+                    newest_bind: AtomicU64::new(0),
+                    segments <- new_mutex!(None),
                 }),
             )?;
             prepare_sgt_segments(dev, gem, &bo, &lease)?;
@@ -2575,7 +2610,10 @@ impl Vm {
         let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease();
         let vm_bo = self
             .gpuvm
-            .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?;
+            .obtain(gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
         prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?;
 
         let mut resources = VmOpResources {
@@ -2829,6 +2867,66 @@ impl Vm {
         Ok(())
     }
 
+    fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result {
+        let _residency = self.enter_retirement_residency();
+        // Removing whole mappings only does unmaps, so no preallocated VAs
+        // are needed for the indexed path.
+        let mut resources = VmOpResources {
+            preallocated_gpuvas: [None, None, None],
+        };
+        let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?;
+
+        // The BO's newest sequence is checked under the same lock as binds. If
+        // it is not newer than the close, every mapping belongs to that close
+        // and the per-object list is safe and substantially cheaper to walk.
+        // SAFETY: See `lock_vm()`.
+        let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } {
+            Err(error) if error == ENOMEM => {
+                return self.drop_mappings_filtered(gem, sequence);
+            }
+            result => result?,
+        };
+        let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else {
+            return Ok(());
+        };
+        if bo.data().newest_bind.load(Ordering::Relaxed) > sequence {
+            drop(vm_exec);
+            drop(bo);
+            return self.drop_mappings_filtered(gem, sequence);
+        }
+
+        mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n");
+        let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo);
+        if result.is_ok() {
+            self.untrack_context_object_before(gem, sequence);
+            self.untrack_shared_object_before(gem, sequence);
+        }
+        drop(vm_exec);
+        drop(bo);
+        if result == Err(ENOMEM) {
+            return self.drop_mappings_filtered(gem, sequence);
+        }
+        result
+    }
+
+    /// A newer binding shares the BO list, so select the closed generation by
+    /// walking the VM without allocating. This is also the ENOMEM fallback.
+    fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result {
+        let mut inner = self.gpuvm.lock_inner();
+        let result = inner.unmap_object_where(
+            gem,
+            |va| va.data().bind_sequence <= sequence,
+            VmInner::unmap_gpuva,
+        );
+        if result.is_ok() {
+            self.untrack_context_object_before(gem, sequence);
+            self.untrack_shared_object_before(gem, sequence);
+        }
+        drop(inner);
+        self.bo_deferred_cleanup();
+        result
+    }
+
     /// Preserve the indexed fast path normally; memory pressure must not lose
     /// accepted object cleanup just because an exec or operation list failed.
     fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result {

drivers/gpu/drm/asahi/mmu/bind.rs

@@ -84,7 +84,10 @@ impl Vm {
         let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease();
         let vm_bo = self
             .gpuvm
-            .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?;
+            .obtain(gem, pin_init!(VmBoData {
+                newest_bind: AtomicU64::new(0),
+                segments <- new_mutex!(None),
+            }))?;
         prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?;
         if (addr | size | offset) & UAT_PGMSK as u64 != 0 {
             return Err(EINVAL);
@@ -153,13 +156,18 @@ impl Vm {
                                 flags,
                             });
                             let retired = result.is_ok().then(|| {
-                                let retired = self.commit_context_binding(&map.gem, &map.context);
+                                let retired = self.commit_context_binding(
+                                    &map.gem,
+                                    &map.context,
+                                    ctx.bind_sequence,
+                                );
                                 self.track_shared_binding(
                                     &map.gem,
                                     map.addr,
                                     map.size,
                                     map.offset,
                                     map.single_page,
+                                    ctx.bind_sequence,
                                 );
                                 retired
                             });

drivers/gpu/drm/asahi/mmu/context.rs

@@ -14,6 +14,7 @@ use kernel::drm::gem::BaseObject;
 struct Binding {
     gem: ARef<gem::Object>,
     offset: usize,
+    bind_sequence: u64,
 }
 struct Bindings {
     generation: u64,
@@ -115,6 +116,7 @@ impl Vm {
         &self,
         gem: &ARef<gem::Object>,
         update: &PreparedBinding,
+        bind_sequence: u64,
     ) -> RetiredBindings {
         let mut retired = RetiredBindings::new();
         if update.touched == 0 {
@@ -136,6 +138,7 @@ impl Vm {
                 *slot = Some(Binding {
                     gem: gem.clone(),
                     offset,
+                    bind_sequence,
                 });
             }
         }
@@ -173,6 +176,10 @@ impl Vm {
     }
 
     pub(crate) fn untrack_context_object(&self, gem: &gem::Object) {
+        self.untrack_context_object_before(gem, u64::MAX);
+    }
+
+    pub(super) fn untrack_context_object_before(&self, gem: &gem::Object, sequence: u64) {
         let Some(context) = self.context_bindings.as_ref() else {
             return;
         };
@@ -181,7 +188,8 @@ impl Vm {
         for slot in &mut bindings.slots {
             if slot
                 .as_ref()
-                .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem))
+                .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)
+                    && binding.bind_sequence <= sequence)
             {
                 *slot = None;
                 touched = true;

drivers/gpu/drm/asahi/mmu/lifetime.rs

@@ -47,6 +47,12 @@ impl DriverMappings {
     }
 }
 
+struct ClosedObject {
+    gem: ARef<gem::Object>,
+    /// Highest binding sequence covered by the close or merged closes.
+    bind_sequence: u64,
+}
+
 struct State {
     active: usize,
     commits: usize,
@@ -57,7 +63,7 @@ struct State {
     cleanup_failed: bool,
     close_ranges: Option<(Range<u64>, Range<u64>)>,
     unmaps: KVec<super::bind::PreparedUserUnmap>,
-    objects: KVec<ARef<gem::Object>>,
+    objects: KVec<ClosedObject>,
 }
 
 #[pin_data]
@@ -578,18 +584,22 @@ impl Vm {
             return self.drop_mappings_now(gem);
         };
         let mut state = lifetime.state.lock();
-        if !state
-            .objects
-            .iter()
-            .any(|object| core::ptr::eq(&**object, gem))
-        {
-            state.objects.push(gem.into(), GFP_KERNEL)?;
+        let bind_sequence = self.current_bind_sequence();
+        if let Some(object) = state.objects.iter_mut().find(|object| {
+            core::ptr::eq(&*object.gem, gem)
+        }) {
+            object.bind_sequence = object.bind_sequence.max(bind_sequence);
+        } else {
+            state.objects.push(ClosedObject {
+                gem: gem.into(),
+                bind_sequence,
+            }, GFP_KERNEL)?;
         }
         if state.active != 0 || state.draining {
             // Stop new jobs from selecting this accepted cleanup candidate.
             // The queued object retains its GEM while the candidate is removed.
-            // Actual unmap repeats this under exec to order any intervening bind.
-            self.untrack_context_object(gem);
+            // Preserve a context view installed by a later bind.
+            self.untrack_context_object_before(gem, bind_sequence);
             return Ok(());
         }
         state.draining = true;
@@ -674,7 +684,7 @@ impl Vm {
                 }
             }
             for object in objects {
-                if let Err(error) = self.drop_mappings_now(&object) {
+                if let Err(error) = self.drop_mappings_before(&object.gem, object.bind_sequence) {
                     failure.get_or_insert(error);
                     pr_err!("MMU: deferred object unmap failed\n");
                 }