From ab481148b183679d3faf423f6bd97b5c3874f6a9 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 10 Nov 2026 21:13:21 -0500 Subject: [PATCH 1/3] rust: drm: gem: shmem: objects over driver-supplied pages Object::new_external() creates a GEM object whose backing is a page array that the object's driver data holds (pages pinned from a process, say) instead of shmem pages. The object is a private GEM object, with no shmem file, so that no C helper can ever allocate pages for it. Its SG table is built and DMA-mapped once, at creation, under a Devres, and unmapped when the object is freed or the device unbound. The mapping never outlives the pages. The driver data is initialized first and the pages are mapped only once it holds them: a driver initializer that fails, and with that releases the pages it was given, leaves nothing mapped, and if the mapping fails the driver data is dropped with no mapping left. When the object is freed, the mapping field is dropped before the driver data. The page array is passed as a raw pointer, since it may be memory the driver data owns, and is only read once that data is initialized. The DMA mapping does no CPU cache maintenance on map or unmap: the pages are the owner's to use through cacheable CPU mappings, and the invalidation an unmap does for a non-coherent device would discard data written through them. The device must therefore reach the pages themselves: a bounce buffer, which an unsynchronized mapping never updates, will not do. The table is mapped with DMA_ATTR_SKIP_CPU_SYNC and DMA_ATTR_NO_BOUNCE, so the DMA layer itself fails the mapping where it would put other memory in a page's place: with EIO for pages it would bounce (those the device cannot address, all pages where bouncing is forced or the device needs unencrypted memory, and pages of an untrusted device whose IOMMU granule is larger than a page), and with EOPNOTSUPP for a device that is mapped by DMA operations of its own. Nothing is inferred from the DMA addresses, so a device with a bus offset, an address encoding or an IOMMU is served like any other. The constructor is unsafe, and its safety contract has two parts. One is the lifetime of the pages. The other is that the device accesses the pages coherently with the CPU caches. The two attributes give the device the pages themselves and leave the caches alone; they make no device coherent, the mapping succeeds for one that is not, and nothing here can check it. The caller has to know it of its hardware. Such objects get their own GEM vtable: no mmap or vm_ops, vmap, pin or get_sg_table, and an export that refuses (the core would otherwise fall back to its generic export). vmap() fails, madvise() is ignored, and the reclaim and purge helpers report the backing as never releasable. Tested: KUnit rust_drm_gem_shmem in QEMU (arm64, 16K pages), 18 of 18 passed, the six new tests included: an object over three pages has a table whose DMA addresses are those pages, no shmem file or pages, and frees without touching the pages' contents; vmap, drm_gem_vmap (EOPNOTSUPP), export (EINVAL), madvise, idle release and purge are refused; no pages, a foreign device or a write-combined config fail with EINVAL; a device whose bus sees every CPU address 1 TiB up (a dma_range_map) gets DMA addresses that are not the pages' physical addresses and its object is created; with a driver whose data owns the pages, no mapping exists at the moment they are released, whether the object is freed or the driver's initializer fails after taking them; and for a device whose bus reaches all memory but one page, an ordinary mapping of that page is a bounce buffer (its DMA address lies outside the page, and the memory there holds what the page held when it was mapped and does not follow a later store to the page), an object over pages the device reaches is created, and an object over a page it reaches and that one fails with EIO, with no mapping left when the driver data releases the pages. The two tests with a bus build the device's dma_range_map from struct bus_dma_region as C defines it: the bindings include linux/dma-direct.h, the DMA layer's own header, for KUnit builds only, and the abstraction uses nothing from it. Developed with Claude Code. Signed-off-by: Erik Bourget --- rust/bindings/bindings_helper.h | 9 + rust/kernel/drm/gem/shmem.rs | 821 +++++++++++++++++++++++++++++++- 2 files changed, 826 insertions(+), 4 deletions(-) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index acabd105e3b1f..01e0e6cfbced8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -233,6 +233,15 @@ const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; #endif +#if IS_ENABLED(CONFIG_KUNIT) +/* + * struct bus_dma_region, for KUnit tests that give a test device a + * dma_range_map. Not for drivers or abstractions: the header is the DMA + * layer's own. + */ +#include +#endif + #if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) #include "../../drivers/android/binder/rust_binder.h" #include "../../drivers/android/binder/rust_binder_events.h" diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fa77f4d3d9f55..aa6b5e17d603d 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -130,6 +130,10 @@ pub struct Object { sgt_lock: Mutex<()>, /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, + /// The device mapping of an object over driver-supplied pages ([`Self::new_external`]). + /// Declared before `inner`, which holds those pages, so that it is unmapped first, and + /// initialized after it, so that nothing is mapped before `inner` holds them. + external: Option>, #[pin] inner: T, } @@ -172,6 +176,30 @@ impl Object { evict: None, }; + /// `drm_gem_object_funcs` vtable for objects over driver-supplied pages. + /// + /// They have no shmem pages for the C helpers to map, pin or export: every CPU, kernel and + /// dma-buf view through the DRM core fails instead (no `mmap` or `vm_ops`, no `vmap`, `pin` + /// or `get_sg_table`, and an `export` that refuses, since without one the core would use its + /// generic export). + const EXTERNAL_VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: None, + export: Some(Self::refuse_export_callback), + pin: None, + unpin: None, + get_sg_table: None, + vmap: None, + vunmap: None, + mmap: None, + status: None, + rss: None, + vm_ops: ptr::null(), + evict: None, + }; + /// Return a raw pointer to the embedded drm_gem_shmem_object. fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { self.obj.get() @@ -213,6 +241,13 @@ impl Object { let _ = unsafe { KBox::from_raw(this) }; } + extern "C" fn refuse_export_callback( + _obj: *mut bindings::drm_gem_object, + _flags: i32, + ) -> *mut bindings::dma_buf { + EINVAL.to_ptr() + } + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> where @@ -222,6 +257,10 @@ impl Object { if self.size() < SIZE { return Err(ENOSPC); } + // The C helper would allocate shmem pages for an object over driver-supplied pages. + if self.is_external() { + return Err(EINVAL); + } let mut map: MaybeUninit = MaybeUninit::uninit(); let guard = DmaResvGuard::new(self); @@ -303,7 +342,10 @@ impl Object { let lease = SGLease { obj: self.into() }; // SAFETY: Import attachment ownership is immutable after publication. - let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + let sgt = if let Some(external) = &self.external { + // Mapped at creation and never rebuilt: once the device is unbound, there is none. + external.access(dev)?.sgt.get() + } else if !unsafe { (*self.as_raw()).import_attach }.is_null() { // Imports belong to the exporter. Never register them for Rust devres cleanup, // rebuild their tables, or release their backing. // SAFETY: The attachment table stays valid until this object is freed. @@ -351,6 +393,7 @@ impl Object { sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), + external: None, inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -388,6 +431,123 @@ impl Object { Ok(obj) } + /// Create a DRM object whose backing is `pages`, which the driver data built from `args` + /// holds, instead of shmem pages. Its size is that of the pages. + /// + /// The object has no shmem file. Its device SG table is built and DMA-mapped here, once, for + /// `parent` (the DRM device's parent), and is unmapped when the object is freed or the device + /// is unbound. The mapping does no CPU cache maintenance, on map or unmap: the pages are the + /// owner's to read and write through cacheable CPU mappings, and invalidating CPU caches when + /// the object is freed would discard data written through them. The device must therefore be + /// coherent with CPU caches for this memory. `writable` selects a bidirectional mapping; + /// otherwise the mapping is for device reads only. + /// + /// The device must reach the pages themselves at the DMA addresses of the table: a bounce + /// buffer, which a mapping without CPU synchronization never updates, will not do. The pages + /// are therefore mapped with `DMA_ATTR_NO_BOUNCE`, and the DMA layer fails the mapping where + /// it would substitute other memory for a page: with `EIO` for pages it would bounce (the + /// device cannot address them, bouncing is forced, or the device needs unencrypted memory), + /// and with `EOPNOTSUPP` for a device that is mapped by DMA operations of its own. `EIO` is + /// the DMA layer's error for any page it cannot map, a refused bounce buffer or not. + /// + /// The driver data is initialized first, and the pages are mapped only once it holds them: if + /// `T::new()` or its initializer fails, nothing was mapped, and if the mapping fails, the + /// driver data is dropped with no mapping left. When the object is freed, the mapping is + /// removed before the driver data is dropped. + /// + /// The object cannot be CPU-mapped or exported through the DRM core, kernel-mapped with + /// [`Self::vmap`], released by idle reclaim or purged: those fail, or report that the backing + /// is not releasable. + /// + /// # Safety + /// + /// - From the moment the initializer `T::new()` returns for `args` has succeeded, and until + /// the driver data it built is dropped, `pages` points to an array of valid pages, and the + /// array and the pages stay allocated and stay that data's backing. `pages` is not + /// dereferenced before that, nor if the initializer fails, so the array may be memory that + /// `args` owns. + /// - The device accesses the pages coherently with the CPU caches, through every mapping the + /// driver gives it of them at the DMA addresses of the table and for as long as the object + /// exists: its next read of a location sees a store made through a cacheable CPU mapping, + /// and the CPU's next load sees a store of the device, with no cache maintenance in + /// between. Nothing here establishes or checks that. `DMA_ATTR_NO_BOUNCE` makes the DMA + /// addresses those of the pages themselves and `DMA_ATTR_SKIP_CPU_SYNC` leaves the caches + /// alone, for a device that is not coherent as for one that is, and whether or not the + /// device is marked DMA-coherent: the mapping succeeds either way. The caller has to know + /// it of its hardware. + pub unsafe fn new_external( + dev: &Device, + parent: &device::Device, + pages: *const [*mut bindings::page], + writable: bool, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + if parent.as_raw() != dev.as_ref().as_ref().as_raw() { + return Err(EINVAL); + } + if config.map_wc || config.reclaimable_cpu_mappings || config.pages_bounded_reclaim { + return Err(EINVAL); + } + if pages.is_empty() { + return Err(EINVAL); + } + let size = pages + .len() + .checked_mul(crate::page::PAGE_SIZE) + .ok_or(EOVERFLOW)?; + + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + sgt_access: AtomicUsize::new(0), + // The driver data takes the pages over, and an initializer that fails may already + // have released them: map them only once it has succeeded. A failure after that + // drops the mapping, if it was made, and then the driver data. + inner <- T::new(dev, size, args), + external: Some(Devres::new( + parent, + // SAFETY: `inner` is initialized, so per the caller's contract `pages` is an + // array of valid pages that stay allocated until `inner` is dropped, which + // comes after this mapping is. The caller answers for the device's coherent + // access to them. + Ok::<_, Error>(unsafe { ExternalDma::new(parent, &*pages, size, writable) }?), + )?), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::EXTERNAL_VTABLE }; + + // A private object has no shmem file, so that no helper can ever allocate pages for it. + // SAFETY: The arguments are all valid via the type invariants; the object is not + // published yet. + unsafe { bindings::drm_gem_private_object_init(dev.as_raw(), new.as_raw(), size) }; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_private_object_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + Ok(obj) + } + + /// Whether the object's backing is driver-supplied pages ([`Self::new_external`]). + pub fn is_external(&self) -> bool { + self.external.is_some() + } + /// `gem_create_object` callback, for the objects the DRM core allocates on the driver's /// behalf (dma-buf imports and dumb buffers) before it initializes their shmem part. /// @@ -409,6 +569,7 @@ impl Object { sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), + external: None, inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -459,7 +620,8 @@ impl Object { /// /// The caller holds the object's DMA reservation lock. pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 1 && + !self.is_external() && + self.sgt_access.load(Ordering::Acquire) == 1 && // SAFETY: The caller establishes the reservation-lock contract. unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } @@ -482,7 +644,8 @@ impl Object { /// /// The caller holds the object's DMA reservation lock. pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 0 && + !self.is_external() && + self.sgt_access.load(Ordering::Acquire) == 0 && // SAFETY: The caller establishes the reservation-lock contract. unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } @@ -498,6 +661,9 @@ impl Object { /// The caller holds this object's DMA reservation lock, has removed and invalidated every /// device mapping, and excludes new device access. pub unsafe fn release_idle_pages_locked(&self) -> Result { + if self.is_external() { + return Err(EBUSY); + } self.sgt_access .compare_exchange(0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| EBUSY)?; @@ -511,8 +677,11 @@ impl Object { /// Apply purge advice (0 = needed, >0 = may be purged) under the /// object's reservation lock. Returns false if the object was already - /// purged; the advice is then ignored. + /// purged; the advice is then ignored. Objects over driver-supplied pages take no advice. pub fn madvise(&self, madv: i32) -> bool { + if self.is_external() { + return true; + } let _guard = DmaResvGuard::new(self); // SAFETY: The reservation lock is held, as the C helper requires. unsafe { bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0 } @@ -536,6 +705,9 @@ impl Object { /// The caller holds this object's DMA reservation lock, has removed and /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { + if self.is_external() { + return Err(EBUSY); + } // SAFETY: The caller's contract; the exclusive state below excludes new leases. if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } @@ -807,6 +979,110 @@ unsafe impl Send for SGTableCleanup {} // SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. unsafe impl Sync for SGTableCleanup {} +/// The device mapping of an object over driver-supplied pages: an SG table over the pages, +/// DMA-mapped without CPU cache maintenance and without bounce buffers. +/// +/// # Invariants +/// +/// `sgt` is an initialized table, DMA-mapped for `dev` with `dir`, over pages that stay valid +/// while this exists. +struct ExternalDma { + sgt: KBox>, + dev: ARef, + dir: bindings::dma_data_direction, +} + +/// The number of live [`ExternalDma`] mappings, for the tests of the order in which a mapping and +/// the owner of its pages go. +#[cfg(CONFIG_KUNIT = "y")] +static EXTERNAL_MAPPINGS: AtomicUsize = AtomicUsize::new(0); + +// SAFETY: The table is only read once built, and the device reference may move between threads. +unsafe impl Send for ExternalDma {} +// SAFETY: See above. +unsafe impl Sync for ExternalDma {} + +impl ExternalDma { + /// The attributes of the mapping. No CPU cache maintenance, on map or unmap: the pages are + /// their owner's to use through cacheable CPU mappings. And no bounce buffer, which such a + /// mapping would never update: the DMA layer fails the mapping instead of substituting other + /// memory for a page. + const ATTRS: c_ulong = + (bindings::DMA_ATTR_SKIP_CPU_SYNC | bindings::DMA_ATTR_NO_BOUNCE) as c_ulong; + + /// # Safety + /// + /// - `pages` are valid pages that stay allocated for the life of the returned mapping. + /// - `dev` accesses them coherently with the CPU caches while the mapping exists, as + /// [`Object::new_external`] requires of its caller: the attributes of this mapping keep + /// the DMA layer from substituting or synchronizing memory, and make no device coherent. + unsafe fn new( + dev: &device::Device, + pages: &[*mut bindings::page], + size: usize, + writable: bool, + ) -> Result { + if pages.is_empty() { + return Err(EINVAL); + } + let sgt = KBox::new(Opaque::::zeroed(), GFP_KERNEL)?; + // SAFETY: `dev` is a valid device. + let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { + 0 => u32::MAX, + max => u32::try_from(max).unwrap_or(u32::MAX), + }; + // SAFETY: `sgt` is zeroed memory for a table, `pages` holds at least one valid page and + // covers `size` bytes. + to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + // The table only reads the array. + pages.as_ptr().cast_mut(), + pages.len().try_into()?, + 0, + size, + max_segment, + GFP_KERNEL.as_raw(), + ) + })?; + let dir = if writable { + bindings::dma_data_direction_DMA_BIDIRECTIONAL + } else { + bindings::dma_data_direction_DMA_TO_DEVICE + }; + // SAFETY: The table was built above; `dev` is bound. + let mapped = to_result(unsafe { + bindings::dma_map_sgtable(dev.as_raw(), sgt.get(), dir, Self::ATTRS) + }); + if let Err(e) = mapped { + // SAFETY: The table was allocated above and is not mapped. + unsafe { bindings::sg_free_table(sgt.get()) }; + return Err(e); + } + #[cfg(CONFIG_KUNIT = "y")] + EXTERNAL_MAPPINGS.fetch_add(1, Ordering::AcqRel); + // INVARIANT: The table is built and mapped. + Ok(Self { + sgt, + dev: dev.into(), + dir, + }) + } +} + +impl Drop for ExternalDma { + fn drop(&mut self) { + // SAFETY: Per the type invariants the table is mapped for `dev` with `dir`; it is unmapped + // and freed once, here. + unsafe { + bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.get(), self.dir, Self::ATTRS); + bindings::sg_free_table(self.sgt.get()); + } + #[cfg(CONFIG_KUNIT = "y")] + EXTERNAL_MAPPINGS.fetch_sub(1, Ordering::AcqRel); + } +} + #[kunit_tests(rust_drm_gem_shmem)] mod tests { use super::*; @@ -1154,6 +1430,543 @@ mod tests { Ok(()) } + fn page_pointers( + pages: &[crate::types::Owned; N], + ) -> [*mut bindings::page; N] { + core::array::from_fn(|i| pages[i].as_ptr()) + } + + fn alloc_pages() -> Result<[crate::types::Owned; N]> { + let mut pages = KVec::with_capacity(N, GFP_KERNEL)?; + for _ in 0..N { + pages.push( + crate::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?, + GFP_KERNEL, + )?; + } + pages.try_into().map_err(|_| EINVAL) + } + + #[test] + fn external_pages_back_the_table() -> Result { + let (dev, drm) = create_drm_dev()?; + // Declared before the object, so they outlive it. + let pages = alloc_pages::<3>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }?; + assert!(obj.is_external()); + assert_eq!(obj.size(), 3 * PAGE_SIZE); + // SAFETY: The object is live; nothing else changes these fields. + let (filp, shmem_pages) = unsafe { ((*obj.as_raw()).filp, (*obj.as_raw_shmem()).pages) }; + assert!(filp.is_null()); + assert!(shmem_pages.is_null()); + let table = obj.sg_table(dev.as_ref().as_ref())?; + // The faux device maps directly: the DMA addresses are the pages' physical addresses. + let mut next = raw.iter(); + let mut total = 0; + for segment in table.iter() { + let mut offset = 0; + while offset < u64::from(segment.dma_len()) { + let page = *next.next().ok_or(EINVAL)?; + // SAFETY: `page` is one of the live test pages. + assert_eq!( + unsafe { bindings::page_to_phys(page) }, + segment.dma_address() + offset + ); + offset += PAGE_SIZE as u64; + } + total += segment.dma_len() as usize; + } + assert_eq!(total, 3 * PAGE_SIZE); + assert!(next.next().is_none()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(table); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + // The pages are the caller's: freeing the object leaves their contents alone. + let marker = 0x5au8; + // SAFETY: The page is live and the byte is within it. + unsafe { pages[1].write_raw(&marker, 7, 1)? }; + drop(obj); + let mut back = 0u8; + // SAFETY: As above. + unsafe { pages[1].read_raw(&mut back, 7, 1)? }; + assert_eq!(back, marker); + Ok(()) + } + + #[test] + fn external_objects_refuse_cpu_views_and_reclaim() -> Result { + let (dev, drm) = create_drm_dev()?; + let pages = alloc_pages::<2>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + false, + ObjectConfig::default(), + (), + ) + }?; + assert_eq!(obj.vmap::<4>().err(), Some(EINVAL)); + let mut map = MaybeUninit::::zeroed(); + // SAFETY: A live object; the core fails without a vmap callback. + assert_eq!( + unsafe { bindings::drm_gem_vmap(obj.as_raw(), map.as_mut_ptr()) }, + -(bindings::EOPNOTSUPP as i32) + ); + // SAFETY: The vtable is static. + let funcs = unsafe { &*(*obj.as_raw()).funcs }; + assert!(funcs.mmap.is_none() && funcs.vm_ops.is_null()); + assert!(funcs.pin.is_none() && funcs.get_sg_table.is_none()); + let export = funcs.export.ok_or(EINVAL)?; + // SAFETY: The callback refuses without touching the object. + let buf = unsafe { export(obj.as_raw(), 0) }; + assert_eq!(buf as isize, -(bindings::EINVAL as isize)); + // Advice is ignored, and leases or none, the backing is never releasable. + assert!(obj.madvise(1)); + let lease = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device mappings exist in this test. + let (madv, releasable, released) = unsafe { + ( + obj.madv_locked(), + obj.sole_sg_lease_releasable_locked(), + obj.release_idle_pages_locked(), + ) + }; + assert_eq!(madv, 0); + assert!(!releasable); + assert_eq!(released, Err(EBUSY)); + } + drop(lease); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: As above, with no lease left. + let (releasable, released, purged) = unsafe { + ( + obj.unleased_pages_releasable_locked(), + obj.release_idle_pages_locked(), + obj.purge_locked(), + ) + }; + assert!(!releasable); + assert_eq!(released, Err(EBUSY)); + assert_eq!(purged, Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn external_objects_need_pages_and_their_device() -> Result { + let (dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"OtherKunit", None)?; + let pages = alloc_pages::<1>()?; + let raw = page_pointers(&pages); + let no_pages: [*mut bindings::page; 0] = []; + // SAFETY: No pages are referenced. + let empty = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &no_pages[..], + true, + ObjectConfig::default(), + (), + ) + }; + assert_eq!(empty.err(), Some(EINVAL)); + // SAFETY: `pages` outlive the attempt. + let wrong = unsafe { + Object::::new_external( + &drm, + reg.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }; + assert_eq!(wrong.err(), Some(EINVAL)); + // SAFETY: As above. + let wc = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig { + map_wc: true, + ..Default::default() + }, + (), + ) + }; + assert_eq!(wc.err(), Some(EINVAL)); + Ok(()) + } + + /// The entry that ends a device's `dma_range_map`, the map that tells how the device's bus sees + /// CPU addresses (firmware describes it for a real device). + const BUS_MAP_END: bindings::bus_dma_region = bindings::bus_dma_region { + cpu_start: 0, + dma_start: 0, + size: 0, + }; + + /// Puts `dev` on the bus that `map`, which ends with [`BUS_MAP_END`], describes. + /// + /// # Safety + /// + /// `dev` is private to the calling test, and has no bus map and no DMA mapping yet. + unsafe fn set_bus_map( + dev: &device::Device, + map: [bindings::bus_dma_region; N], + ) -> Result { + let map = KBox::new(map, GFP_KERNEL)?; + // SAFETY: Per the safety requirements nothing else uses this field. The driver core + // frees the map with kfree() when the device goes. + unsafe { + (*dev.as_raw()).dma_range_map = KBox::into_raw(map).cast::() + }; + Ok(()) + } + + #[test] + fn external_mapping_follows_a_bus_offset() -> Result { + // The bus of this device sees every CPU address 1 TiB up. + const OFFSET: u64 = 1 << 40; + let (dev, drm) = create_drm_dev()?; + let parent: &device::Device = dev.as_ref().as_ref(); + // SAFETY: The faux device is private to this test and has no mapping yet. + unsafe { + set_bus_map( + parent, + [ + bindings::bus_dma_region { + cpu_start: 0, + dma_start: OFFSET, + size: OFFSET, + }, + BUS_MAP_END, + ], + )? + }; + let pages = alloc_pages::<3>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + parent, + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }?; + // The DMA layer applied the offset: no DMA address is its page's physical address. The + // device reaches the pages themselves all the same, and the object was accepted. + let table = obj.sg_table(parent)?; + let mut next = raw.iter(); + for segment in table.iter() { + let mut offset = 0; + while offset < u64::from(segment.dma_len()) { + let page = *next.next().ok_or(EINVAL)?; + // SAFETY: `page` is one of the live test pages. + assert_eq!( + unsafe { bindings::page_to_phys(page) } + OFFSET, + segment.dma_address() + offset + ); + offset += PAGE_SIZE as u64; + } + } + assert!(next.next().is_none()); + Ok(()) + } + + // A second fake driver, whose object data owns the pages the object is created over (as a + // driver that pins memory for an object does) and whose initializer can fail after taking + // them. + + /// The number of device mappings of external objects that existed when a [`PageOwner`] last + /// let go of its pages. + static MAPPED_AT_RELEASE: AtomicUsize = AtomicUsize::new(usize::MAX); + + struct PageOwner { + _pages: [crate::types::Owned; 2], + fail: bool, + } + + impl Drop for PageOwner { + fn drop(&mut self) { + MAPPED_AT_RELEASE.store(EXTERNAL_MAPPINGS.load(Ordering::Acquire), Ordering::Release); + } + } + + struct OwnerDriver; + struct OwnerFile; + #[pin_data] + struct OwnerObject { + _owner: PageOwner, + } + + impl drm::file::DriverFile for OwnerFile { + type Driver = OwnerDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Err(ENODEV) + } + + fn as_raw(&self) -> *mut bindings::drm_file { + unreachable!("KUnit object tests do not open DRM files") + } + } + + #[vtable] + impl gem::DriverObject for OwnerObject { + type Driver = OwnerDriver; + type Args = PageOwner; + + fn new( + _dev: &drm::Device, + _size: usize, + args: Self::Args, + ) -> impl PinInit { + let fail = args.fail; + // A failure unwinds this initializer, which drops the owner it already took. + try_pin_init!(OwnerObject { + _owner: args, + _: { + if fail { + return Err(ENOMEM); + } + }, + }) + } + } + + #[vtable] + impl drm::Driver for OwnerDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = OwnerFile; + type Object = Object; + type ParentDevice = faux::Device; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + // The pages go with the driver data. Whether the object is freed or its creation fails in the + // driver's initializer, no device mapping of them is left at that moment. + #[test] + fn external_mapping_never_outlives_the_pages() -> Result { + let reg = faux::Registration::new(c"OwnerKunit", None)?; + let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: As in `create_drm_dev()`. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } + let drm = UnregisteredDevice::::new(fdev, try_pin_init!(KunitData {}))?; + for fail in [true, false] { + let pages = alloc_pages::<2>()?; + let raw = page_pointers(&pages); + MAPPED_AT_RELEASE.store(usize::MAX, Ordering::Release); + // SAFETY: The driver data holds the pages, and `raw` outlives the call. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: pages, + fail, + }, + ) + }; + if fail { + assert_eq!(obj.err(), Some(ENOMEM)); + } else { + let obj = obj?; + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 1); + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), usize::MAX); + drop(obj); + } + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), 0); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + } + Ok(()) + } + + // A device that cannot address one of an object's pages. Left to itself the DMA layer gives + // such a device a bounce buffer in the page's place, which the test first shows with an + // ordinary mapping of that page. An object over the page must fail instead, with nothing + // left mapped when its pages are released. + // + // Built for arm64, which always sets up bounce buffers (it bounces small kmalloc() buffers + // through them), so that the ordinary mapping has one to get. + #[cfg(CONFIG_ARM64)] + #[test] + fn external_mapping_refuses_a_bounce_buffer() -> Result { + let reg = faux::Registration::new(c"BounceKunit", None)?; + let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + let reachable = alloc_pages::<2>()?; + // The page the device cannot address is the lower of two, and comes second in its + // object: the two never form one segment, and the first is mapped before the second + // fails. + let [a, b] = alloc_pages::<2>()?; + let (near, hidden) = if a.phys() > b.phys() { (a, b) } else { (b, a) }; + let start = hidden.phys(); + let end = start + PAGE_SIZE as u64; + // SAFETY: As in `create_drm_dev()`. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } + // The bus of this device reaches all memory, at its CPU address, except `hidden`. + // SAFETY: The faux device is private to this test and has no mapping yet. + unsafe { + set_bus_map( + dev, + [ + bindings::bus_dma_region { + cpu_start: end, + dma_start: end, + size: u64::MAX - end, + }, + bindings::bus_dma_region { + cpu_start: 0, + dma_start: 0, + size: start, + }, + BUS_MAP_END, + ], + )? + }; + let drm = UnregisteredDevice::::new(fdev, try_pin_init!(KunitData {}))?; + + // An ordinary mapping of `hidden` for this device succeeds, and is a bounce buffer: its + // DMA address, which for this bus is a CPU address, lies outside the page, the memory + // there holds what the page held when it was mapped, and a later store to the page + // does not show there. + let (before, after) = (0xa5u8, 0x5au8); + // SAFETY: The page is live and private to this test, as it is below. + unsafe { hidden.write_raw(&before, 0, 1)? }; + let sgt = KBox::new(Opaque::::zeroed(), GFP_KERNEL)?; + let one = [hidden.as_ptr()]; + // SAFETY: `sgt` is zeroed memory for a table and `one` holds one valid page. + to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + one.as_ptr().cast_mut(), + 1, + 0, + PAGE_SIZE, + u32::MAX, + GFP_KERNEL.as_raw(), + ) + })?; + let dir = bindings::dma_data_direction_DMA_TO_DEVICE; + // SAFETY: The table was built above; `dev` is bound. + let mapped = + to_result(unsafe { bindings::dma_map_sgtable(dev.as_raw(), sgt.get(), dir, 0) }); + let mut bounce = None; + if mapped.is_ok() { + // SAFETY: The table is mapped until the unmap below. + let addr = unsafe { scatterlist::SGTable::from_raw(sgt.get()) } + .iter() + .next() + .map(|segment| segment.dma_address()); + // SAFETY: See above. + let stored = unsafe { hidden.write_raw(&after, 0, 1) }; + if let (Some(addr), Ok(())) = (addr, stored) { + let base = addr & !(PAGE_SIZE as u64 - 1); + let mut seen = 0u8; + // SAFETY: The memory the DMA layer mapped stays allocated until the unmap, and + // nothing else uses it. + let read = unsafe { + crate::page::Page::borrow_phys(&base) + .map(|page| page.read_raw(&mut seen, (addr - base) as usize, 1)) + }; + if let Some(Ok(())) = read { + bounce = Some((addr, seen)); + } + } + // SAFETY: The table was mapped above for `dev` with `dir`. + unsafe { bindings::dma_unmap_sgtable(dev.as_raw(), sgt.get(), dir, 0) }; + } + // SAFETY: The table was allocated above and is not mapped. + unsafe { bindings::sg_free_table(sgt.get()) }; + assert_eq!(mapped, Ok(())); + let (addr, seen) = bounce.ok_or(EINVAL)?; + assert!(addr < start || addr >= end); + assert_eq!(seen, before); + + // The device takes an object over pages it reaches. + let raw = page_pointers(&reachable); + // SAFETY: The driver data holds the pages, and `raw` outlives the call. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: reachable, + fail: false, + }, + ) + }?; + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 1); + drop(obj); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + + // An object over `hidden` fails: the DMA layer refuses the mapping instead of bouncing + // the page. The driver data, which had taken the pages, releases them with no mapping + // left, the first page's included. + let raw = [near.as_ptr(), hidden.as_ptr()]; + MAPPED_AT_RELEASE.store(usize::MAX, Ordering::Release); + // SAFETY: As above. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: [near, hidden], + fail: false, + }, + ) + }; + assert_eq!(obj.err(), Some(EIO)); + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), 0); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + Ok(()) + } + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { From ecc94a2151f79919c4099ee74ecbaea5b8eaa93b Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 10 Nov 2026 21:13:22 -0500 Subject: [PATCH 2/3] drm/asahi: GEM_USERPTR, buffer objects over the caller's memory DRM_IOCTL_ASAHI_GEM_USERPTR creates a GEM object whose backing is a page-aligned range of the caller's address space, without a copy, for Vulkan's VK_EXT_external_memory_host. GET_PARAMS reports it with DRM_ASAHI_FEATURE_USERPTR (bit 37), on G17 GPUs. The range is pinned at creation with pin_user_pages_fast(FOLL_LONGTERM, plus FOLL_WRITE unless DRM_ASAHI_GEM_USERPTR_READ_ONLY), all of it or none, and stays pinned until the object is freed; unpinning marks the pages dirty if the GPU could write them. The pages count against RLIMIT_MEMLOCK through mm->pinned_vm unless the caller has CAP_IPC_LOCK, and their size is admitted against the client and device memory limits and counted as held for the object's life (not added to the process's RSS again: they already are its memory). The core mm handles the rest: fork gives the child copies of pinned anonymous pages, munmap leaves the pinned pages to the object, writable pins of read-only mappings and of write-tracked file mappings, and pins of device or PFN mappings, fail with EFAULT, and read-only pins of private mappings unshare the pages first. A read-only shared file mapping pins its page-cache pages, which stay resident while the object lives. User-memory objects are only ever mapped by user VM binds: a read-only object takes only GPU read-only binds (EACCES otherwise), since its pages may be a file the caller cannot write. Kernel and firmware mappings (timestamp buffers, kernel aliases) refuse them, as do MMAP_OFFSET, MADVISE, the VM status mirror (no vmap) and export. They are never idle-reclaim or purge candidates. The pages are mapped for the GPU without CPU cache maintenance, and the caller and the GPU use them at the same time. That rests on an assumption about the hardware, which neither the DMA attributes nor the driver establish: that the GPU's accesses through user VM binds are coherent with the CPU caches for normal cacheable memory. The driver relies on it already: objects created with DRM_ASAHI_GEM_WRITEBACK are mapped cacheable into their process and used by the GPU through user VM binds with the same page attributes, with no cache maintenance by the driver and no UAPI call for it, and a user-memory object is bound the same way. The assumption covers normal cacheable RAM only: pinning refuses device and PFN mappings but does not check the caching attributes of every mapping it accepts. It was measured on the J716c (below); other machines with a G17 GPU were not measured. Tested: KUnit in QEMU (arm64, 16K pages), 545 tests, 543 passed, 2 skipped (unrelated configuration), including the new asahi_gem_user_memory suite against a real test mm: pinning four anonymous pages counts four in pinned_vm and drops it again, and a store through the user mapping is read back from the pinned page; unaligned, empty and overflowing ranges fail, a range running into an unmapped page fails after pinning part of it and leaves pinned_vm unchanged; a read-only mapping refuses a writable pin and takes a read-only one; a pinned page keeps its contents after munmap. On the J716c (M5 Max MacBook Pro), with a Vulkan driver that implements VK_EXT_external_memory_host on this ioctl, dEQP-VK.memory.external_memory_host.* gives 3 passed, 8 not supported and none failed. There, a Vulkan program in which the CPU stores a new pattern to one imported range and a marker to another through its own mappings, a compute job copies the first to the second and the CPU compares, found no wrong word: 20000 iterations each with ranges of 16 KiB and 64 KiB, 3000 with 1 MiB and 30 with 64 MiB. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/gpu/drm/asahi/driver.rs | 2 + drivers/gpu/drm/asahi/file.rs | 104 ++++++++ drivers/gpu/drm/asahi/gem.rs | 411 +++++++++++++++++++++++++++++- drivers/gpu/drm/asahi/mmu.rs | 5 +- drivers/gpu/drm/asahi/mmu/bind.rs | 2 +- include/uapi/drm/asahi_drm.h | 88 +++++++ 6 files changed, 607 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 2b8bdac35f43d..52c018cbd1dc1 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -140,6 +140,8 @@ impl drm::driver::Driver for AsahiDrmDriver { ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), (ASAHI_GEM_MADVISE, drm_asahi_gem_madvise, ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), + (ASAHI_GEM_USERPTR, drm_asahi_gem_userptr, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_userptr), } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 3c86eb47f31c9..9fe3de9109731 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -565,6 +565,9 @@ impl File { gpu.get_params(&mut params)?; params.features |= gpu.feature_flags(); + if Self::supports_user_memory(reg_data) { + params.features |= uapi::DRM_ASAHI_FEATURE_USERPTR as u64; + } let size = core::mem::size_of::().min(data.size.try_into()?); @@ -897,6 +900,103 @@ impl File { Ok(0) } + /// Whether this GPU takes objects over user memory (DRM_IOCTL_ASAHI_GEM_USERPTR). + /// + /// The bindings of G17 user VMs are the only ones that admit such an object's pages. They + /// are GPU-only page-table entries, and the GPU's accesses through those are taken to be + /// coherent with the CPU caches, as they are for the cacheable CPU mappings of ordinary + /// objects (DRM_ASAHI_GEM_WRITEBACK): see `gem::new_user_memory_object()`. + fn supports_user_memory(reg_data: &driver::AsahiDrmRegistrationData<'_>) -> bool { + reg_data.gpu.as_any().is::() + } + + /// IOCTL: gem_userptr: Create a GEM object over the caller's own memory. + pub(crate) fn gem_userptr( + device: &AsahiDrmDevice, + reg_data: &driver::AsahiDrmRegistrationData<'_>, + data: &mut uapi::drm_asahi_gem_userptr, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_userptr addr={:#x} size={:#x}\n", + file.inner().id, + data.userptr, + data.size + ); + + if !Self::supports_user_memory(reg_data) { + return Err(EOPNOTSUPP); + } + let private = data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0; + if data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_USERPTR_READ_ONLY) + != 0 + || (!private && data.vm_id != 0) + || data.pad != 0 + || data.size == 0 + || (data.userptr | data.size) & (mmu::UAT_PGMSK | (kernel::page::PAGE_SIZE - 1)) as u64 + != 0 + { + cls_pr_debug!(Errors, "gem_userptr: Invalid arguments\n"); + return Err(EINVAL); + } + let addr = usize::try_from(data.userptr)?; + let size = gem::checked_object_size(data.size.try_into()?)?; + let writable = data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_USERPTR_READ_ONLY == 0; + + let resv_gem; + let resv_obj = if private { + let vms = file.inner().vms(); + let vms = vms.lock(); + resv_gem = vms + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .vm + .get_resv_obj(); + Some(resv_gem.deref()) + } else { + None + }; + + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_userptr: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + let pages = gem::UserPages::pin(addr, size, writable).inspect_err(|e| { + cls_pr_debug!(Errors, "gem_userptr: pinning failed: {:?}\n", e); + })?; + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let gem = gem::new_user_memory_object( + device, + parent.as_ref(), + pages, + data.flags, + resv_obj, + commit, + )?; + drop(registered); + + data.handle = gem.create_handle(file)?; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_userptr size={:#x} handle={:#x?}\n", + file.inner().id, + size, + data.handle + ); + + Ok(0) + } + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. pub(crate) fn gem_mmap_offset( device: &AsahiDrmDevice, @@ -917,6 +1017,10 @@ impl File { } let gem = gem::Object::lookup_handle(file, data.handle)?; + // The caller maps a user-memory object's memory itself. + if gem.is_user_memory() { + return Err(EINVAL); + } data.offset = gem.create_mmap_offset()?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 854e3636e1d0c..3c3b667e61af5 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,6 +21,7 @@ use kernel::{ }, // }, error::Result, + mm::Mm, new_mutex, prelude::*, sync::aref::ARef, @@ -30,6 +31,7 @@ use kernel::{ use core::ops::Range; use core::sync::atomic::{ + AtomicI64, AtomicU64, AtomicUsize, Ordering, // @@ -63,6 +65,9 @@ pub(crate) struct AsahiObject { kernel_mappings: AtomicU64, /// Memory accounting and process attribution of the held backing. charge: memacct::Charge, + /// The pinned pages of a user-memory object (DRM_IOCTL_ASAHI_GEM_USERPTR), its backing. + /// The shmem layer unmaps them from the device before this is dropped. + user_memory: Option, /// Serializes overlapping timestamp writes through every binding of this BO. #[pin] pub(crate) timestamp_lock: Mutex<()>, @@ -113,9 +118,42 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { /// Takes the SG table of `gem`, which holds its pages for the device, and marks the /// object held (see [`set_mapping_held`]). +/// +/// For mappings the driver creates for itself or the firmware (kernel aliases, timestamp +/// buffers, firmware structures), which user-memory objects never back: those are refused. pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, +) -> Result> { + if gem.user_memory.is_some() { + return Err(EINVAL); + } + mapped_sg_table(gem, dev) +} + +/// Takes the SG table of `gem` for a user VM binding with `prot`, as [`held_sg_table`]. +/// +/// A read-only user-memory object takes only GPU read-only bindings (`EACCES` otherwise): its +/// pages may be another file's page cache, which the caller may not write. +pub(crate) fn held_user_sg_table<'a>( + gem: &'a Object, + dev: &'a kernel::device::Device, + prot: mmu::Prot, +) -> Result> { + if gem.user_memory.as_ref().is_some_and(|user| !user.writable) + && prot != mmu::PROT_GPU_SHARED_RO + { + return Err(EACCES); + } + mapped_sg_table(gem, dev) +} + +/// Takes the SG table of `gem` for a device mapping that [`held_sg_table`] or +/// [`held_user_sg_table`] already admitted (the caller holds a lease from one of them, or the +/// mapping exists). +pub(crate) fn mapped_sg_table<'a>( + gem: &'a Object, + dev: &'a kernel::device::Device, ) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); @@ -232,6 +270,7 @@ pub(crate) struct AsahiObjConfig { exportable: bool, kernel: bool, charge: memacct::Charge, + user_memory: Option, } /// Create a new kernel-owned GEM object. @@ -295,6 +334,7 @@ fn new_kernel_object_mapped( exportable: false, kernel: true, charge, + user_memory: None, }, )?; @@ -335,6 +375,7 @@ pub(crate) fn new_object( exportable: parent_object.is_none(), kernel: false, charge: memacct::Charge::user_current(commit), + user_memory: None, }, )?; @@ -342,6 +383,207 @@ pub(crate) fn new_object( Ok(gem) } +/// Create a user-memory object over `pages`, the caller's memory pinned by [`UserPages::pin`]. +/// +/// `commit` is the admission of the pages' size for the creating client. The pages count as held +/// by the GPU for the object's whole life, but are not attributed to the process again: they are +/// already part of its memory (anonymous memory or page cache it maps). The object cannot be +/// exported, CPU-mapped through the driver, vmapped, purged or idle-reclaimed. +pub(crate) fn new_user_memory_object( + dev: &AsahiDrmDevice, + parent: &kernel::device::Device, + pages: UserPages, + flags: u32, + parent_object: Option<&shmem::Object>, + commit: memacct::Commit, +) -> Result> { + if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() + { + return Err(EINVAL); + } + let writable = pages.writable; + // The page array is a heap allocation: moving `pages` into the object leaves it in place. + let array = core::ptr::slice_from_raw_parts(pages.pages.as_ptr(), pages.pages.len()); + // SAFETY: + // - `array` is the array of `pages`, which the object's data holds from its initialization + // on: the array stays allocated and its pages pinned until `pages` is dropped with that + // data. + // - Coherent access. The object is only ever mapped by user VM binds + // (`held_user_sg_table()`; `held_sg_table()` refuses it for every mapping the driver makes + // for itself or for the firmware), and only on the GPUs of `File::supports_user_memory()`, + // where a user VM bind is a set of GPU-only entries (`mmu::PROT_GPU_SHARED_*`) of the VM's + // page tables at the DMA addresses of the pages. The GPU reads and writes the pages + // through those entries and in no other way. This driver depends on such accesses being + // coherent with the CPU caches for its ordinary objects already: an object created with + // DRM_ASAHI_GEM_WRITEBACK is mapped cacheable into its process, which stores to it and + // loads from it while the GPU uses it through entries of the same kind, the driver can do + // no cache maintenance for those accesses, and the UAPI has no call for it. Whether the + // device is marked DMA-coherent does not enter into it: where it is not, the DMA layer + // would clean and invalidate CPU caches when it maps and unmaps, which is what the + // object's mapping skips. + let gem = unsafe { + shmem::Object::::new_external( + dev, + parent, + array, + writable, + shmem::ObjectConfig:: { + parent_resv_obj: parent_object, + ..Default::default() + }, + AsahiObjConfig { + flags, + exportable: false, + kernel: false, + charge: memacct::Charge::user(commit, None), + user_memory: Some(pages), + }, + ) + }?; + gem.charge.set_held(true); + mod_pr_debug!("AsahiObject new user-memory object: id={}\n", gem.id); + Ok(gem) +} + +/// Pages of the calling process pinned as the backing of a user-memory object, and counted in its +/// mm's `pinned_vm` against RLIMIT_MEMLOCK. +pub(crate) struct UserPages { + pages: KVVec<*mut kernel::bindings::page>, + writable: bool, + /// The mm whose `pinned_vm` counts the pages; it may outlive the address space. + mm: ARef, + /// Pages counted in `pinned_vm`: all of the range, including any not pinned yet. + counted: i64, +} + +// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe. +unsafe impl Send for UserPages {} +// SAFETY: Shared access only reads the array. +unsafe impl Sync for UserPages {} + +/// Pages pinned per `pin_user_pages_fast()` call. +const PIN_CHUNK: usize = 16384; + +impl UserPages { + /// Pins the `size` bytes at `addr` in the calling process, for device writes as well if + /// `writable`. Both must be multiples of the page size, and `size` non-zero. + /// + /// All of the range is pinned, long-term, or none of it. Fails with `EFAULT` when a page cannot + /// be pinned: unmapped, a device or PFN mapping, or a writable pin of a mapping that is not + /// writable or whose filesystem tracks writes (`FOLL_LONGTERM` refuses those). Fails with + /// `ENOMEM` when the pages would take the process past RLIMIT_MEMLOCK without CAP_IPC_LOCK, + /// counting every page pinned this way or by other long-term pinners. + /// + /// Anonymous pages pinned here stay with this process across fork(): the child gets copies. + /// Pages unmapped by the process stay pinned, and backing the object, until it is freed. + pub(crate) fn pin(addr: usize, size: usize, writable: bool) -> Result { + let page_size = kernel::page::PAGE_SIZE; + if size == 0 || (addr | size) & (page_size - 1) != 0 { + return Err(EINVAL); + } + addr.checked_add(size).ok_or(EFAULT)?; + let count = size / page_size; + let mm: ARef = kernel::current!() + .mm() + .map(|mm| ARef::from(&**mm)) + .ok_or(EFAULT)?; + let pinned_vm = Self::pinned_vm(&mm); + let added = i64::try_from(count).map_err(|_| ENOMEM)?; + let total = pinned_vm + .fetch_add(added, Ordering::AcqRel) + .saturating_add(added); + if !memlock_allows(total, memlock_limit_pages()) { + // SAFETY: `capable()` has no preconditions. + if !unsafe { kernel::bindings::capable(kernel::bindings::CAP_IPC_LOCK as i32) } { + pinned_vm.fetch_sub(added, Ordering::AcqRel); + return Err(ENOMEM); + } + } + // From here on, dropping `this` unpins what it holds and uncounts all `count` pages. + let mut this = Self { + pages: KVVec::new(), + writable, + mm, + counted: added, + }; + this.pages.reserve(count, GFP_KERNEL)?; + let mut flags = kernel::bindings::FOLL_LONGTERM; + if writable { + flags |= kernel::bindings::FOLL_WRITE; + } + while this.pages.len() < count { + let done = this.pages.len(); + let chunk = (count - done).min(PIN_CHUNK); + // SAFETY: The array has room for `count` entries, of which `done` are in use; the + // call fills at most `chunk` more. + let ret = unsafe { + kernel::bindings::pin_user_pages_fast( + (addr + done * page_size) as _, + chunk as _, + flags, + this.pages.as_mut_ptr().add(done), + ) + }; + if ret <= 0 { + return Err(if ret < 0 { + Error::from_errno(ret) + } else { + EFAULT + }); + } + let ret = (ret as usize).min(chunk); + // SAFETY: The call stored `ret` valid page pointers after the first `done`. + unsafe { this.pages.inc_len(ret) }; + } + Ok(this) + } + + fn pinned_vm(mm: &Mm) -> &AtomicI64 { + // SAFETY: `pinned_vm` is an aligned atomic64 of the live mm, which the kernel only changes + // atomically. + unsafe { + AtomicI64::from_ptr(core::ptr::addr_of_mut!( + (*mm.as_raw()).__bindgen_anon_1.pinned_vm.counter + )) + } + } +} + +impl Drop for UserPages { + fn drop(&mut self) { + if !self.pages.is_empty() { + // SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet + // unpinned. Pages the device may have written are marked dirty. + unsafe { + kernel::bindings::unpin_user_pages_dirty_lock( + self.pages.as_mut_ptr(), + self.pages.len() as _, + self.writable, + ) + }; + } + Self::pinned_vm(&self.mm).fetch_sub(self.counted, Ordering::AcqRel); + } +} + +/// RLIMIT_MEMLOCK of the calling process, in pages. +fn memlock_limit_pages() -> u64 { + let task = kernel::current!().as_ptr(); + // SAFETY: The current task and its signal structure are live; the limit is read once, as + // `rlimit()` does. + let bytes = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!( + (*(*task).signal).rlim[kernel::bindings::RLIMIT_MEMLOCK as usize].rlim_cur + )) + }; + (bytes >> kernel::page::PAGE_SHIFT) as u64 +} + +/// Whether `pinned` pages stay within a memlock limit of `limit` pages. +fn memlock_allows(pinned: i64, limit: u64) -> bool { + u64::try_from(pinned).is_ok_and(|pinned| pinned <= limit) +} + #[vtable] impl DriverObject for AsahiObject { type Driver = AsahiDrmDriver; @@ -361,6 +603,7 @@ impl DriverObject for AsahiObject { kernel: args.kernel, kernel_mappings: AtomicU64::new(0), charge: args.charge, + user_memory: args.user_memory, timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } @@ -376,6 +619,7 @@ impl DriverObject for AsahiObject { exportable: true, kernel: false, charge: memacct::Charge::none(), + user_memory: None, }) } @@ -442,7 +686,15 @@ impl AsahiObject { } pub(crate) fn idle_reclaim_candidate(&self) -> bool { - !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 + !self.kernel + && !self.exportable + && self.user_memory.is_none() + && self.kernel_mappings.load(Ordering::Acquire) == 0 + } + + /// Whether this object is backed by the creating process's own memory. + pub(crate) fn is_user_memory(&self) -> bool { + self.user_memory.is_some() } } @@ -466,7 +718,10 @@ pub(crate) fn prepare_new_object( /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 || gem.exportable { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 + || gem.exportable + || gem.user_memory.is_some() + { return Err(EINVAL); } Ok(u32::from(gem.madvise(i32::from(dontneed)))) @@ -493,6 +748,158 @@ mod allocation_policy_tests { } } +#[kernel::macros::kunit_tests(asahi_gem_user_memory)] +mod user_memory_tests { + use super::*; + use kernel::{ + bindings, + page::PAGE_SIZE, + uaccess::{UserPtr, UserSlice}, + }; + + const PROT_READ: u64 = 1; + const PROT_WRITE: u64 = 2; + const MAP_PRIVATE: u64 = 2; + const MAP_ANONYMOUS: u64 = 0x20; + + /// Maps `len` bytes of anonymous memory into the test's user address space. + fn map(len: usize, prot: u64) -> Result { + // SAFETY: Called on a KUnit test thread. KUnit attaches a user mm for the test and + // removes the mapping with it when the test ends. + let addr = unsafe { + bindings::kunit_vm_mmap( + bindings::kunit_get_current_test(), + core::ptr::null_mut(), + 0, + len as _, + prot as _, + (MAP_PRIVATE | MAP_ANONYMOUS) as _, + 0, + ) + } as usize; + if addr == 0 || addr > usize::MAX - 4095 { + return Err(ENOMEM); + } + Ok(addr) + } + + fn pinned_vm() -> Result { + let mm: ARef = kernel::current!() + .mm() + .map(|mm| ARef::from(&**mm)) + .ok_or(EINVAL)?; + Ok(UserPages::pinned_vm(&mm).load(Ordering::Acquire)) + } + + fn read_pinned(pages: &UserPages, index: usize, offset: usize) -> Result { + let page = *pages.pages.get(index).ok_or(EINVAL)?; + // SAFETY: The page is pinned while `pages` lives; the word is inside it. + Ok(unsafe { + let p = bindings::kmap_local_page(page); + let value = core::ptr::read_volatile(p.cast::().add(offset).cast::()); + bindings::kunmap_local(p); + value + }) + } + + #[test] + fn memlock_limit_policy() { + assert!(memlock_allows(0, 0)); + assert!(memlock_allows(512, 512)); + assert!(!memlock_allows(513, 512)); + assert!(!memlock_allows(-1, 512)); + assert!(memlock_allows(i64::MAX, u64::MAX)); + } + + #[test] + fn pins_anonymous_memory() -> Result { + let addr = map(4 * PAGE_SIZE, PROT_READ | PROT_WRITE)?; + let before = pinned_vm()?; + let pages = UserPages::pin(addr, 4 * PAGE_SIZE, true)?; + assert_eq!(pages.pages.len(), 4); + assert_eq!(pinned_vm()?, before + 4); + // CPU writes through the caller's mapping land in the pinned pages. + UserSlice::new(UserPtr::from_addr(addr + PAGE_SIZE + 8), 4) + .writer() + .write_slice(&0x1234_5678u32.to_ne_bytes())?; + assert_eq!(read_pinned(&pages, 1, 8)?, 0x1234_5678); + drop(pages); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn bad_ranges_pin_nothing() -> Result { + let addr = map(3 * PAGE_SIZE, PROT_READ | PROT_WRITE)?; + // SAFETY: Unmaps the last page of the test's own mapping. + assert_eq!( + unsafe { bindings::vm_munmap((addr + 2 * PAGE_SIZE) as _, PAGE_SIZE) }, + 0 + ); + let before = pinned_vm()?; + assert_eq!( + UserPages::pin(addr + 1, PAGE_SIZE, true).err(), + Some(EINVAL) + ); + assert_eq!(UserPages::pin(addr, 0, true).err(), Some(EINVAL)); + assert_eq!( + UserPages::pin(addr, PAGE_SIZE + 1, true).err(), + Some(EINVAL) + ); + // Two pages pin before the hole: those are unpinned again. + assert_eq!( + UserPages::pin(addr, 3 * PAGE_SIZE, true).err(), + Some(EFAULT) + ); + assert_eq!( + UserPages::pin(addr + 2 * PAGE_SIZE, PAGE_SIZE, false).err(), + Some(EFAULT) + ); + let top = usize::MAX & !(PAGE_SIZE - 1); + assert_eq!( + UserPages::pin(top, 2 * PAGE_SIZE, false).err(), + Some(EFAULT) + ); + assert_eq!(pinned_vm()?, before); + // What remains of the mapping still pins. + drop(UserPages::pin(addr, 2 * PAGE_SIZE, true)?); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn read_only_mappings_pin_read_only() -> Result { + let addr = map(2 * PAGE_SIZE, PROT_READ)?; + let before = pinned_vm()?; + assert_eq!( + UserPages::pin(addr, 2 * PAGE_SIZE, true).err(), + Some(EFAULT) + ); + assert_eq!(pinned_vm()?, before); + let pages = UserPages::pin(addr, 2 * PAGE_SIZE, false)?; + assert_eq!(pages.pages.len(), 2); + assert!(!pages.writable); + assert_eq!(pinned_vm()?, before + 2); + drop(pages); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn pages_outlive_the_mapping() -> Result { + let addr = map(PAGE_SIZE, PROT_READ | PROT_WRITE)?; + UserSlice::new(UserPtr::from_addr(addr + 16), 4) + .writer() + .write_slice(&0xa5a5_5a5au32.to_ne_bytes())?; + let pages = UserPages::pin(addr, PAGE_SIZE, true)?; + // SAFETY: Unmaps the test's own mapping; the pinned page stays allocated. + assert_eq!(unsafe { bindings::vm_munmap(addr as _, PAGE_SIZE) }, 0); + assert_eq!(read_pinned(&pages, 0, 16)?, 0xa5a5_5a5a); + drop(pages); + Ok(()) + } +} + /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ad7f6273141ed..3ac6e589a6393 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2382,8 +2382,9 @@ impl HandoffFlush { } } +/// Copies the DMA segments of `gem`, whose device mapping a caller's lease already admitted. fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { - let sgt = crate::gem::held_sg_table(gem, dev)?; + let sgt = crate::gem::mapped_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { @@ -2996,7 +2997,7 @@ impl Vm { self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let _bind_lease = crate::gem::held_user_sg_table(gem, dev, prot)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index a792cdd1bec08..9748357561077 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -89,7 +89,7 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let sg_lease = crate::gem::held_user_sg_table(gem, dev, prot)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 3a0bf7ae28b58..c51b239fb4917 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -96,6 +96,12 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ DRM_ASAHI_GEM_MADVISE, + + /** + * @DRM_ASAHI_GEM_USERPTR: Create a buffer object over the caller's own + * memory (DRM_ASAHI_FEATURE_USERPTR). + */ + DRM_ASAHI_GEM_USERPTR, }; #define DRM_ASAHI_MAX_CLUSTERS 64 @@ -489,6 +495,12 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34) +/* + * DRM_ASAHI_FEATURE_USERPTR: DRM_IOCTL_ASAHI_GEM_USERPTR is available. A + * feature bit like those of enum drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_USERPTR ((1ULL) << 37) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -671,6 +683,12 @@ enum drm_asahi_gem_flags { * @DRM_ASAHI_GEM_VM_PRIVATE: BO is private to this GPU VM (no exports). */ DRM_ASAHI_GEM_VM_PRIVATE = (1L << 1), + + /** + * @DRM_ASAHI_GEM_USERPTR_READ_ONLY: The GPU only reads the memory of a + * DRM_IOCTL_ASAHI_GEM_USERPTR object. Only that ioctl accepts it. + */ + DRM_ASAHI_GEM_USERPTR_READ_ONLY = (1L << 2), }; /** @@ -695,6 +713,75 @@ struct drm_asahi_gem_create { __u32 pad; }; +/** + * struct drm_asahi_gem_userptr - Arguments passed to + * DRM_IOCTL_ASAHI_GEM_USERPTR + * + * Creates a buffer object whose backing is the caller's memory at + * [@userptr, @userptr + @size), without a copy: the CPU accesses it through + * the caller's own mapping, and the GPU through VM_BIND mappings of the + * object, as for any other object. Available when DRM_ASAHI_FEATURE_USERPTR + * is set; otherwise the ioctl fails with EOPNOTSUPP or EINVAL. + * + * The pages of the range are pinned when the object is created and stay + * pinned until it is freed: all of them or, on failure, none. They are the + * pages the range maps at creation; later changes to the caller's mappings + * (munmap, mremap, a new mapping over the range, truncating a mapped file) + * leave the object on those pages, which stay allocated until it is freed. + * After fork(), the child gets copies of the pinned anonymous pages; the + * object stays the parent's memory. The CPU and GPU views are coherent + * without cache maintenance. + * + * The pinned pages count against the caller's RLIMIT_MEMLOCK, together with + * its other long-term pins, unless it has CAP_IPC_LOCK (ENOMEM otherwise), + * and their size against the client and device limits of + * drm_asahi_params_global::client_memory_limit and + * drm_asahi_params_global::memory_limit (ENOMEM). They are never released by + * memory-pressure reclaim. + * + * Without DRM_ASAHI_GEM_USERPTR_READ_ONLY, the whole range must be mapped + * writable, and not be a shared mapping of a file whose filesystem tracks + * writes. With it, any readable mapping is accepted, and the object takes + * only GPU read-only VM_BIND mappings (a mapping with DRM_ASAHI_BIND_WRITE + * fails with EACCES). Pages of private mappings, file mappings included, are + * pinned as the caller's own copies, as a write would make them: a private + * file mapping is copied out of the page cache. A read-only shared file + * mapping pins the page-cache pages themselves, which then stay resident + * while the object lives. Device and PFN mappings, including CPU mappings of + * other buffer objects, are refused (EFAULT). + * + * The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported, + * given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with + * DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all + * EINVAL). + */ +struct drm_asahi_gem_userptr { + /** + * @userptr: Start of the range, aligned to both the CPU page size and + * the GPU page size (16 KiB). + */ + __u64 userptr; + + /** @size: Size of the range, a non-zero multiple of both page sizes. */ + __u64 size; + + /** + * @flags: DRM_ASAHI_GEM_VM_PRIVATE and DRM_ASAHI_GEM_USERPTR_READ_ONLY. + */ + __u32 flags; + + /** + * @vm_id: VM ID to assign to the BO, if DRM_ASAHI_GEM_VM_PRIVATE is set + */ + __u32 vm_id; + + /** @handle: Returned GEM handle for the BO */ + __u32 handle; + + /** @pad: MBZ */ + __u32 pad; +}; + /** * struct drm_asahi_gem_mmap_offset - Arguments passed to * DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET @@ -2193,6 +2280,7 @@ enum { DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOCTL_ASAHI(W, QUEUE_DESTROY, queue_destroy), DRM_IOCTL_ASAHI_SUBMIT = DRM_IOCTL_ASAHI(W, SUBMIT, submit), DRM_IOCTL_ASAHI_GEM_MADVISE = DRM_IOCTL_ASAHI(WR, GEM_MADVISE, gem_madvise), + DRM_IOCTL_ASAHI_GEM_USERPTR = DRM_IOCTL_ASAHI(WR, GEM_USERPTR, gem_userptr), }; #if defined(__cplusplus) From 7e2760fcd0bbd4bdb9abc89d1ba38b0ea9d6c1a1 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 11 Nov 2026 12:37:21 -0500 Subject: [PATCH 3/3] selftests: drm/asahi: GEM_USERPTR A kselftest harness program for the render node that exercises DRM_IOCTL_ASAHI_GEM_USERPTR without submitting GPU work: - anonymous memory: creation counts the pages in VmPin and the size in the client's drm-total-system, a read-write bind succeeds, MMAP_OFFSET and MADVISE fail with EINVAL, export and a timestamp BIND_OBJECT fail, munmap while bound keeps the pin until the handle closes, and closing it gives everything back; - bad ranges: unaligned address or size, zero size, unknown flags and a write-back flag fail with EINVAL; a range with a hole fails with EFAULT and leaves VmPin as it was, as does the CPU mapping of another buffer object (a PFN mapping), writable or read-only; - a read-only shared mapping of /usr/bin/sh: a writable import fails with EFAULT, a read-only one succeeds and takes only read binds (EACCES for a write bind); - RLIMIT_MEMLOCK: the test gives up CAP_IPC_LOCK first, so whoever runs it a request above the limit must fail with ENOMEM and leave VmPin as it was; a request within the limit succeeds, and a further one fails when the pages already pinned take it past the limit. The soft limit it sets is what the process has pinned plus 16 pages, or as many as the hard limit has room for (skipped below two); - after fork the child's write lands in its own copy. The driver keeps ranges of every VM for its own mappings, where a bind fails with EINVAL, and GET_PARAMS does not list them: the test binds at the first address, in steps of 4 GiB, at which an ordinary buffer object binds. The program is listed in the selftests' TARGETS. It skips where renderD128 is not this driver's node or lacks the feature, and where the caller has neither CAP_IPC_LOCK nor a RLIMIT_MEMLOCK hard limit with room for the four pages a test pins (the soft limit is raised to it if need be). Tested on the J716c (M5 Max MacBook Pro), built there with gcc against this tree's UAPI header, as an unprivileged user and as root: pass 5, fail 0, skip 0, also with RLIMIT_MEMLOCK at 128 KiB (soft and hard) and at 32 KiB soft, 128 KiB hard. With a hard limit of 48 KiB the unprivileged run skips all five and root passes five; with 16 KiB root passes four and skips memlock_limit. Developed with Claude Code. Signed-off-by: Erik Bourget --- tools/testing/selftests/Makefile | 1 + .../drivers/gpu/asahi-userptr/Makefile | 4 + .../drivers/gpu/asahi-userptr/userptr.c | 434 ++++++++++++++++++ 3 files changed, 439 insertions(+) create mode 100644 tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 6fb32e0396fa9..5ec4864f5eb22 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -25,6 +25,7 @@ TARGETS += drivers/dma-buf TARGETS += drivers/gpu/apple-afkv2 TARGETS += drivers/gpu/apple-dcp TARGETS += drivers/gpu/asahi-m4 +TARGETS += drivers/gpu/asahi-userptr TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice TARGETS += drivers/net diff --git a/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile b/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile new file mode 100644 index 0000000000000..1bd98ee6745dc --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CFLAGS += -O2 -Wall $(KHDR_INCLUDES) +TEST_GEN_PROGS := userptr +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c b/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c new file mode 100644 index 0000000000000..abb5ba2f5402c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * DRM_IOCTL_ASAHI_GEM_USERPTR on the GPU's render node: what it accepts and + * refuses, and the accounting (VmPin in /proc/self/status, the client's + * drm-total-system in its fdinfo). No GPU work is submitted. Skipped where + * the render node is not this driver's or lacks the feature, and where the + * caller has neither CAP_IPC_LOCK nor a RLIMIT_MEMLOCK hard limit with room + * for the four pages a test pins. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../kselftest_harness.h" + +#define PAGE 0x4000UL + +static int ioc(int fd, unsigned long cmd, void *arg) +{ + int ret; + + do { + ret = ioctl(fd, cmd, arg); + } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + return ret ? -errno : 0; +} + +/* VmPin of this process (mm->pinned_vm) in kB. */ +static long vm_pin_kb(void) +{ + char line[256]; + long kb = -1; + FILE *f = fopen("/proc/self/status", "r"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %ld kB", &kb) == 1) + break; + fclose(f); + return kb; +} + +/* drm-total-system of the client behind @fd, in bytes: the objects it created. */ +static long long client_total(int fd) +{ + char path[64], line[256], unit[8]; + long long value = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", fd); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unit[0] = 0; + if (sscanf(line, "drm-total-system: %lld %7s", &value, unit) < 1) + continue; + if (!strcmp(unit, "KiB")) + value <<= 10; + else if (!strcmp(unit, "MiB")) + value <<= 20; + break; + } + fclose(f); + return value; +} + +static int userptr(int fd, void *ptr, uint64_t size, uint32_t flags, + uint32_t vm, uint32_t *handle) +{ + struct drm_asahi_gem_userptr args = { + .userptr = (uintptr_t)ptr, + .size = size, + .flags = flags, + .vm_id = (flags & DRM_ASAHI_GEM_VM_PRIVATE) ? vm : 0, + }; + int ret = ioc(fd, DRM_IOCTL_ASAHI_GEM_USERPTR, &args); + + if (!ret) + *handle = args.handle; + return ret; +} + +static int bind(int fd, uint32_t vm, uint32_t handle, uint64_t addr, + uint64_t size, uint32_t flags) +{ + struct drm_asahi_gem_bind_op op = { + .flags = flags, .handle = handle, .range = size, .addr = addr, + }; + struct drm_asahi_vm_bind args = { + .vm_id = vm, .num_binds = 1, .stride = sizeof(op), + .userptr = (uintptr_t)&op, + }; + + return ioc(fd, DRM_IOCTL_ASAHI_VM_BIND, &args); +} + +static int unbind(int fd, uint32_t vm, uint64_t addr, uint64_t size) +{ + return bind(fd, vm, 0, addr, size, DRM_ASAHI_BIND_UNBIND); +} + +static void close_handle(int fd, uint32_t handle) +{ + struct drm_gem_close args = { .handle = handle }; + + ioc(fd, DRM_IOCTL_GEM_CLOSE, &args); +} + +/* Whether the driver behind @fd is this one. */ +static int is_asahi(int fd) +{ + char name[16] = ""; + struct drm_version version = { + .name = name, .name_len = sizeof(name) - 1, + }; + + return !ioc(fd, DRM_IOCTL_VERSION, &version) && !strcmp(name, "aurora"); +} + +/* + * Clears CAP_IPC_LOCK in the effective set of this process, which is always + * allowed. Returns 0 once the process is without it. + */ +static int drop_cap_ipc_lock(void) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + }; + struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3]; + + if (syscall(SYS_capget, &header, data)) + return -errno; + data[CAP_TO_INDEX(CAP_IPC_LOCK)].effective &= ~CAP_TO_MASK(CAP_IPC_LOCK); + return syscall(SYS_capset, &header, data) ? -errno : 0; +} + +/* The largest range a test binds, and the most one pins outside memlock_limit. */ +#define BIND_MAX (4 * PAGE) + +/* + * Whether this process may pin BIND_MAX more: it has CAP_IPC_LOCK, or + * RLIMIT_MEMLOCK has room for it, after raising the soft limit to the hard + * one if that takes it. + */ +static int memlock_room(void) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + }; + struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3]; + long pin = vm_pin_kb(); + uint64_t need = (pin > 0 ? (uint64_t)pin << 10 : 0) + BIND_MAX; + struct rlimit lim; + + if (!syscall(SYS_capget, &header, data) && + (data[CAP_TO_INDEX(CAP_IPC_LOCK)].effective & + CAP_TO_MASK(CAP_IPC_LOCK))) + return 1; + if (getrlimit(RLIMIT_MEMLOCK, &lim)) + return 0; + if (lim.rlim_cur == RLIM_INFINITY || lim.rlim_cur >= need) + return 1; + if (lim.rlim_max != RLIM_INFINITY && lim.rlim_max < need) + return 0; + lim.rlim_cur = lim.rlim_max; + return !setrlimit(RLIMIT_MEMLOCK, &lim); +} + +/* + * A GPU address at which BIND_MAX bytes can be bound. The driver keeps + * ranges of every VM for its own mappings, where a bind fails with EINVAL, + * and GET_PARAMS does not list them: take the first address, in steps of + * 4 GiB below the kernel range, at which the ordinary buffer object @handle + * (of BIND_MAX bytes) binds. + */ +static uint64_t find_va(int fd, uint32_t vm, uint32_t handle, uint64_t start, + uint64_t end) +{ + uint64_t va; + + for (va = start + (4ULL << 30); va + BIND_MAX <= end; va += 4ULL << 30) { + if (bind(fd, vm, handle, va, BIND_MAX, DRM_ASAHI_BIND_READ)) + continue; + return unbind(fd, vm, va, BIND_MAX) ? 0 : va; + } + return 0; +} + +FIXTURE(asahi) +{ + int fd; + struct drm_asahi_params_global params; + uint32_t vm; + uint64_t va; +}; + +FIXTURE_SETUP(asahi) +{ + struct drm_asahi_get_params get = { + .pointer = (uintptr_t)&self->params, + .size = sizeof(self->params), + }; + + self->fd = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC); + if (self->fd < 0 || !is_asahi(self->fd)) + SKIP(return, "renderD128 is not an asahi render node"); + memset(&self->params, 0, sizeof(self->params)); + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GET_PARAMS, &get)); + if (!(self->params.features & DRM_ASAHI_FEATURE_USERPTR)) + SKIP(return, "no DRM_ASAHI_FEATURE_USERPTR"); + if (!memlock_room()) + SKIP(return, "RLIMIT_MEMLOCK has no room for the %lu bytes a test pins", + BIND_MAX); + + uint64_t kernel = self->params.vm_kernel_min_size > (32ULL << 30) ? + self->params.vm_kernel_min_size : 32ULL << 30; + struct drm_asahi_vm_create vm = { + .kernel_start = self->params.vm_end - kernel, + .kernel_end = self->params.vm_end, + }; + /* Lives as long as the file, so that the client's total stays put. */ + struct drm_asahi_gem_create probe = { + .size = BIND_MAX, .flags = DRM_ASAHI_GEM_VM_PRIVATE, + }; + + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_VM_CREATE, &vm)); + self->vm = vm.vm_id; + probe.vm_id = self->vm; + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_CREATE, &probe)); + self->va = find_va(self->fd, self->vm, probe.handle, + self->params.vm_start, vm.kernel_start); + ASSERT_NE(0, self->va); +} + +FIXTURE_TEARDOWN(asahi) +{ + close(self->fd); +} + +TEST_F(asahi, anonymous_memory_round_trip) +{ + const uint64_t size = 4 * PAGE; + long pin0 = vm_pin_kb(); + long long total0 = client_total(self->fd); + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint32_t h = 0; + + ASSERT_NE(MAP_FAILED, mem); + ASSERT_GE(total0, 0); + ASSERT_EQ(0, userptr(self->fd, mem, size, DRM_ASAHI_GEM_VM_PRIVATE, + self->vm, &h)); + EXPECT_EQ(pin0 + (long)(size >> 10), vm_pin_kb()); + EXPECT_EQ(total0 + (long long)size, client_total(self->fd)); + EXPECT_EQ(0, bind(self->fd, self->vm, h, self->va, size, + DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE)); + + /* No CPU mapping, advice, export or special binding through DRM. */ + struct drm_asahi_gem_mmap_offset off = { .handle = h }; + struct drm_asahi_gem_madvise madv = { + .handle = h, .madv = DRM_ASAHI_GEM_MADV_DONTNEED, + }; + struct drm_prime_handle prime = { .handle = h, .flags = DRM_CLOEXEC }; + struct drm_asahi_gem_bind_object ts = { + .op = DRM_ASAHI_BIND_OBJECT_OP_BIND, + .flags = DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS, + .handle = h, .range = PAGE, + }; + + EXPECT_EQ(-EINVAL, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, &off)); + EXPECT_EQ(-EINVAL, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MADVISE, &madv)); + EXPECT_NE(0, ioc(self->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime)); + EXPECT_NE(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_BIND_OBJECT, &ts)); + + /* Unmapped while bound: the object keeps the pages until it goes. */ + ASSERT_EQ(0, munmap(mem, size)); + EXPECT_EQ(pin0 + (long)(size >> 10), vm_pin_kb()); + EXPECT_EQ(0, unbind(self->fd, self->vm, self->va, size)); + close_handle(self->fd, h); + EXPECT_EQ(pin0, vm_pin_kb()); + EXPECT_EQ(total0, client_total(self->fd)); +} + +TEST_F(asahi, bad_ranges) +{ + char *mem = mmap(NULL, 3 * PAGE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + long pin0 = vm_pin_kb(); + uint32_t h = 0; + + ASSERT_NE(MAP_FAILED, mem); + ASSERT_EQ(0, munmap(mem + 2 * PAGE, PAGE)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem + 4096, PAGE, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE + 4096, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, 0, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE, 1u << 0 /* WRITEBACK */, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE, 1u << 7, 0, &h)); + /* Two pages pin, the third is a hole: nothing stays pinned. */ + EXPECT_EQ(-EFAULT, userptr(self->fd, mem, 3 * PAGE, 0, 0, &h)); + EXPECT_EQ(pin0, vm_pin_kb()); + /* A CPU mapping of another buffer object is a PFN mapping. */ + struct drm_asahi_gem_create bo = { .size = PAGE }; + struct drm_asahi_gem_mmap_offset off = { 0 }; + void *bo_map; + + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_CREATE, &bo)); + off.handle = bo.handle; + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, &off)); + bo_map = mmap(NULL, PAGE, PROT_READ | PROT_WRITE, MAP_SHARED, + self->fd, off.offset); + ASSERT_NE(MAP_FAILED, bo_map); + EXPECT_EQ(-EFAULT, userptr(self->fd, bo_map, PAGE, 0, 0, &h)); + EXPECT_EQ(-EFAULT, userptr(self->fd, bo_map, PAGE, + DRM_ASAHI_GEM_USERPTR_READ_ONLY, 0, &h)); + munmap(bo_map, PAGE); + close_handle(self->fd, bo.handle); + EXPECT_EQ(pin0, vm_pin_kb()); + munmap(mem, 2 * PAGE); +} + +/* A read-only shared mapping of a file the caller cannot write. */ +TEST_F(asahi, read_only_file_pages) +{ + const char *path = "/usr/bin/sh"; + int file = open(path, O_RDONLY | O_CLOEXEC); + uint32_t h = 0; + + ASSERT_GE(file, 0); + if (lseek(file, 0, SEEK_END) < (off_t)(2 * PAGE)) + SKIP(return, "%s is smaller than two pages", path); + void *mem = mmap(NULL, 2 * PAGE, PROT_READ, MAP_SHARED, file, 0); + + ASSERT_NE(MAP_FAILED, mem); + EXPECT_EQ(-EFAULT, userptr(self->fd, mem, 2 * PAGE, 0, 0, &h)); + ASSERT_EQ(0, userptr(self->fd, mem, 2 * PAGE, + DRM_ASAHI_GEM_USERPTR_READ_ONLY, 0, &h)); + EXPECT_EQ(-EACCES, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE)); + EXPECT_EQ(-EACCES, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_WRITE)); + EXPECT_EQ(0, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_READ)); + EXPECT_EQ(0, unbind(self->fd, self->vm, self->va, 2 * PAGE)); + close_handle(self->fd, h); + munmap(mem, 2 * PAGE); + close(file); +} + +/* + * RLIMIT_MEMLOCK bounds the pages of a caller without CAP_IPC_LOCK. The test + * gives the capability up first (the harness runs each test in a process of + * its own), so a request above the limit has to fail whoever runs it. It + * lowers the soft limit to what the process has pinned already plus 16 pages, + * or as many as the hard limit has room for, and is skipped where that is + * less than two. + */ +TEST_F(asahi, memlock_limit) +{ + struct rlimit old, lim; + long pin0 = vm_pin_kb(); + uint64_t pinned = pin0 > 0 ? (uint64_t)pin0 << 10 : 0; + uint64_t pages = 16, size; + uint32_t h = 0; + void *mem; + + ASSERT_EQ(0, drop_cap_ipc_lock()); + ASSERT_EQ(0, getrlimit(RLIMIT_MEMLOCK, &old)); + if (old.rlim_max != RLIM_INFINITY) { + uint64_t room = old.rlim_max > pinned ? + (old.rlim_max - pinned) / PAGE : 0; + + if (room < pages) + pages = room; + } + if (pages < 2) + SKIP(return, "no room for two pages in the RLIMIT_MEMLOCK hard limit of %llu bytes", + (unsigned long long)old.rlim_max); + size = 4 * pages * PAGE; + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, mem); + lim.rlim_cur = pinned + pages * PAGE; + lim.rlim_max = old.rlim_max; + ASSERT_EQ(0, setrlimit(RLIMIT_MEMLOCK, &lim)); + ASSERT_EQ(-ENOMEM, userptr(self->fd, mem, size, 0, 0, &h)); + EXPECT_EQ(pin0, vm_pin_kb()); + EXPECT_EQ(0, userptr(self->fd, mem, pages / 2 * PAGE, 0, 0, &h)); + /* The pages already pinned count against the limit too. */ + EXPECT_EQ(-ENOMEM, userptr(self->fd, (char *)mem + pages / 2 * PAGE, + pages * PAGE, 0, 0, &h)); + close_handle(self->fd, h); + setrlimit(RLIMIT_MEMLOCK, &old); + munmap(mem, size); +} + +/* After fork the parent keeps its pinned pages; the child writes a copy. */ +TEST_F(asahi, fork_keeps_the_parent_pages) +{ + uint32_t *mem = mmap(NULL, PAGE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint32_t h = 0; + int status; + + ASSERT_NE(MAP_FAILED, mem); + mem[0] = 0x1234; + ASSERT_EQ(0, userptr(self->fd, mem, PAGE, 0, 0, &h)); + pid_t pid = fork(); + + ASSERT_GE(pid, 0); + if (pid == 0) { + mem[0] = 0xdead; + _exit(mem[0] == 0xdead ? 0 : 1); + } + ASSERT_EQ(pid, waitpid(pid, &status, 0)); + EXPECT_EQ(0, WEXITSTATUS(status)); + EXPECT_EQ(0x1234, mem[0]); + close_handle(self->fd, h); + munmap(mem, PAGE); +} + +TEST_HARNESS_MAIN