diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 6baaa97977f5a..3572f26ec7dad 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -366,6 +366,7 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, + bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -424,6 +425,11 @@ struct VmBinding { /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { + /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. + /// + /// Map steps update this monotonically under the VM reservation before + /// linking the GPUVA. Remap pieces keep their original sequence. + newest_bind: AtomicU64, #[pin] segments: Mutex>, } @@ -479,6 +485,9 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, + + /// Sequence assigned by the successful map for its related metadata. + bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -491,6 +500,7 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, + bind_sequence: 0, }) } @@ -521,6 +531,7 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, + bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -539,6 +550,11 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let bind_sequence = self.bind_sequence.fetch_update( + Ordering::AcqRel, + Ordering::Acquire, + |value| value.checked_add(1), + ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -611,12 +627,18 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; + op.vm_bo() + .data() + .newest_bind + .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), + bind_sequence, }, ); + ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -642,6 +664,7 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; + let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -707,9 +730,11 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, ); @@ -994,6 +1019,7 @@ pub(crate) struct Vm { dev: ARef, id: u64, mapping_epoch: Option>, + bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -2112,6 +2138,7 @@ impl Vm { } else { None }; + let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -2124,6 +2151,7 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), + bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -2140,6 +2168,7 @@ impl Vm { id, gpuvm, mapping_epoch, + bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -2304,6 +2333,10 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } + fn current_bind_sequence(&self) -> u64 { + self.bind_sequence.load(Ordering::Acquire) + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2404,7 +2437,8 @@ impl Vm { let bo = self.gpuvm.obtain( gem, pin_init!(VmBoData { - segments <- new_mutex!(None) + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), }), )?; prepare_sgt_segments(dev, gem, &bo, &lease)?; @@ -2492,7 +2526,8 @@ impl Vm { let bo = self.gpuvm.obtain( gem, pin_init!(VmBoData { - segments <- new_mutex!(None) + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), }), )?; prepare_sgt_segments(dev, gem, &bo, &lease)?; @@ -2575,7 +2610,10 @@ impl Vm { let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { @@ -2829,6 +2867,66 @@ impl Vm { Ok(()) } + fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { + let _residency = self.enter_retirement_residency(); + // Removing whole mappings only does unmaps, so no preallocated VAs + // are needed for the indexed path. + let mut resources = VmOpResources { + preallocated_gpuvas: [None, None, None], + }; + let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; + + // The BO's newest sequence is checked under the same lock as binds. If + // it is not newer than the close, every mapping belongs to that close + // and the per-object list is safe and substantially cheaper to walk. + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + Err(error) if error == ENOMEM => { + return self.drop_mappings_filtered(gem, sequence); + } + result => result?, + }; + let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { + return Ok(()); + }; + if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { + drop(vm_exec); + drop(bo); + return self.drop_mappings_filtered(gem, sequence); + } + + mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); + let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(vm_exec); + drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_filtered(gem, sequence); + } + result + } + + /// A newer binding shares the BO list, so select the closed generation by + /// walking the VM without allocating. This is also the ENOMEM fallback. + fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_object_where( + gem, + |va| va.data().bind_sequence <= sequence, + VmInner::unmap_gpuva, + ); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 0fa3f304af1d4..acce5677b9f92 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -84,7 +84,10 @@ impl Vm { let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); @@ -153,13 +156,18 @@ impl Vm { flags, }); let retired = result.is_ok().then(|| { - let retired = self.commit_context_binding(&map.gem, &map.context); + let retired = self.commit_context_binding( + &map.gem, + &map.context, + ctx.bind_sequence, + ); self.track_shared_binding( &map.gem, map.addr, map.size, map.offset, map.single_page, + ctx.bind_sequence, ); retired }); diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index d459a3a91afce..0b11394aa408c 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -14,6 +14,7 @@ use kernel::drm::gem::BaseObject; struct Binding { gem: ARef, offset: usize, + bind_sequence: u64, } struct Bindings { generation: u64, @@ -115,6 +116,7 @@ impl Vm { &self, gem: &ARef, update: &PreparedBinding, + bind_sequence: u64, ) -> RetiredBindings { let mut retired = RetiredBindings::new(); if update.touched == 0 { @@ -136,6 +138,7 @@ impl Vm { *slot = Some(Binding { gem: gem.clone(), offset, + bind_sequence, }); } } @@ -173,6 +176,10 @@ impl Vm { } pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { + self.untrack_context_object_before(gem, u64::MAX); + } + + pub(super) fn untrack_context_object_before(&self, gem: &gem::Object, sequence: u64) { let Some(context) = self.context_bindings.as_ref() else { return; }; @@ -181,7 +188,8 @@ impl Vm { for slot in &mut bindings.slots { if slot .as_ref() - .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) + .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem) + && binding.bind_sequence <= sequence) { *slot = None; touched = true; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 8cf37b37e2e0a..66d507c2d7694 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -47,6 +47,12 @@ impl DriverMappings { } } +struct ClosedObject { + gem: ARef, + /// Highest binding sequence covered by the close or merged closes. + bind_sequence: u64, +} + struct State { active: usize, commits: usize, @@ -57,7 +63,7 @@ struct State { cleanup_failed: bool, close_ranges: Option<(Range, Range)>, unmaps: KVec, - objects: KVec>, + objects: KVec, } #[pin_data] @@ -578,18 +584,22 @@ impl Vm { return self.drop_mappings_now(gem); }; let mut state = lifetime.state.lock(); - if !state - .objects - .iter() - .any(|object| core::ptr::eq(&**object, gem)) - { - state.objects.push(gem.into(), GFP_KERNEL)?; + let bind_sequence = self.current_bind_sequence(); + if let Some(object) = state.objects.iter_mut().find(|object| { + core::ptr::eq(&*object.gem, gem) + }) { + object.bind_sequence = object.bind_sequence.max(bind_sequence); + } else { + state.objects.push(ClosedObject { + gem: gem.into(), + bind_sequence, + }, GFP_KERNEL)?; } if state.active != 0 || state.draining { // Stop new jobs from selecting this accepted cleanup candidate. // The queued object retains its GEM while the candidate is removed. - // Actual unmap repeats this under exec to order any intervening bind. - self.untrack_context_object(gem); + // Preserve a context view installed by a later bind. + self.untrack_context_object_before(gem, bind_sequence); return Ok(()); } state.draining = true; @@ -674,7 +684,7 @@ impl Vm { } } for object in objects { - if let Err(error) = self.drop_mappings_now(&object) { + if let Err(error) = self.drop_mappings_before(&object.gem, object.bind_sequence) { failure.get_or_insert(error); pr_err!("MMU: deferred object unmap failed\n"); } diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index a0c05e3d5c303..fec7a6b61ad8f 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -14,6 +14,7 @@ struct Binding { end: u64, object: usize, offset: u64, + bind_sequence: u64, } #[pin_data] @@ -43,6 +44,7 @@ impl Vm { size: u64, offset: u64, single_page: bool, + bind_sequence: u64, ) { let Some(shared) = &self.shared_bindings else { return; @@ -63,6 +65,7 @@ impl Vm { end, object: core::ptr::from_ref(gem) as usize, offset, + bind_sequence, }, GFP_KERNEL, ); @@ -86,6 +89,10 @@ impl Vm { } pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { + self.untrack_shared_object_before(gem, u64::MAX); + } + + pub(super) fn untrack_shared_object_before(&self, gem: &gem::Object, sequence: u64) { let Some(shared) = &self.shared_bindings else { return; }; @@ -94,7 +101,7 @@ impl Vm { } let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); - bindings.retain(|entry| entry.object != object); + bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index b6aa52428a4e6..3b4f9c5e50221 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -3258,7 +3258,7 @@ drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *vm_bo, void *priv) { struct drm_gpuva_ops *ops; struct drm_gpuva_op *op; - int ret; + int ret = 0; if (unlikely(!vm_bo->vm)) return -EINVAL; diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index d873eb4ef6384..8b40308767619 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -761,24 +761,38 @@ impl GpuVmInnerGuard<'_, T> { /// node or operation list is allocated. Call [`GpuVm::deferred_cleanup`] after releasing this /// guard, including when a callback failed after earlier mappings were removed. pub fn unmap_all(&mut self, unmap: impl FnMut(&mut T, &GpuVa) -> Result) -> Result { - self.unmap_matching(None, unmap) + self.unmap_matching(None, |_| true, unmap) } /// Allocation-free whole-node removal for one GEM object in an immediate-mode VM. /// /// Uses the same translation-removal and deferred-put contract as [`Self::unmap_all`]. This - /// scans the VM list and is intended as a low-memory cleanup fallback. + /// scans the VM list and can be used when the per-object operation list is unavailable. pub fn unmap_object( &mut self, object: &T::Object, unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { - self.unmap_matching(Some(object), unmap) + self.unmap_object_where(object, |_| true, unmap) + } + + /// Remove only mappings of an object selected by `matches`, without allocating. + /// + /// This scans the whole VM and uses the same translation-removal and deferred-put contract as + /// [`Self::unmap_all`]. If a callback fails, mappings already visited remain removed. + pub fn unmap_object_where( + &mut self, + object: &T::Object, + matches: impl FnMut(&GpuVa) -> bool, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(Some(object), matches, unmap) } fn unmap_matching( &mut self, object: Option<&T::Object>, + mut matches: impl FnMut(&GpuVa) -> bool, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { let vm = self.gpuvm.as_raw(); @@ -802,7 +816,11 @@ impl GpuVmInnerGuard<'_, T> { if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { continue; } - unmap(&mut *self.gpuvm.data.get(), GpuVa::::from_raw(va))?; + let typed_va = GpuVa::::from_raw(va); + if !matches(typed_va) { + continue; + } + unmap(&mut *self.gpuvm.data.get(), typed_va)?; // Remove it as `OpUnmap::remove()` does. let op = bindings::drm_gpuva_op_unmap { va, keep: false }; bindings::drm_gpuva_unmap(&op);