diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 2b8bdac35f43d..52c018cbd1dc1 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -140,6 +140,8 @@ impl drm::driver::Driver for AsahiDrmDriver { ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), (ASAHI_GEM_MADVISE, drm_asahi_gem_madvise, ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), + (ASAHI_GEM_USERPTR, drm_asahi_gem_userptr, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_userptr), } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 3c86eb47f31c9..9fe3de9109731 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -565,6 +565,9 @@ impl File { gpu.get_params(&mut params)?; params.features |= gpu.feature_flags(); + if Self::supports_user_memory(reg_data) { + params.features |= uapi::DRM_ASAHI_FEATURE_USERPTR as u64; + } let size = core::mem::size_of::().min(data.size.try_into()?); @@ -897,6 +900,103 @@ impl File { Ok(0) } + /// Whether this GPU takes objects over user memory (DRM_IOCTL_ASAHI_GEM_USERPTR). + /// + /// The bindings of G17 user VMs are the only ones that admit such an object's pages. They + /// are GPU-only page-table entries, and the GPU's accesses through those are taken to be + /// coherent with the CPU caches, as they are for the cacheable CPU mappings of ordinary + /// objects (DRM_ASAHI_GEM_WRITEBACK): see `gem::new_user_memory_object()`. + fn supports_user_memory(reg_data: &driver::AsahiDrmRegistrationData<'_>) -> bool { + reg_data.gpu.as_any().is::() + } + + /// IOCTL: gem_userptr: Create a GEM object over the caller's own memory. + pub(crate) fn gem_userptr( + device: &AsahiDrmDevice, + reg_data: &driver::AsahiDrmRegistrationData<'_>, + data: &mut uapi::drm_asahi_gem_userptr, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_userptr addr={:#x} size={:#x}\n", + file.inner().id, + data.userptr, + data.size + ); + + if !Self::supports_user_memory(reg_data) { + return Err(EOPNOTSUPP); + } + let private = data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0; + if data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_USERPTR_READ_ONLY) + != 0 + || (!private && data.vm_id != 0) + || data.pad != 0 + || data.size == 0 + || (data.userptr | data.size) & (mmu::UAT_PGMSK | (kernel::page::PAGE_SIZE - 1)) as u64 + != 0 + { + cls_pr_debug!(Errors, "gem_userptr: Invalid arguments\n"); + return Err(EINVAL); + } + let addr = usize::try_from(data.userptr)?; + let size = gem::checked_object_size(data.size.try_into()?)?; + let writable = data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_USERPTR_READ_ONLY == 0; + + let resv_gem; + let resv_obj = if private { + let vms = file.inner().vms(); + let vms = vms.lock(); + resv_gem = vms + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .vm + .get_resv_obj(); + Some(resv_gem.deref()) + } else { + None + }; + + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_userptr: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + let pages = gem::UserPages::pin(addr, size, writable).inspect_err(|e| { + cls_pr_debug!(Errors, "gem_userptr: pinning failed: {:?}\n", e); + })?; + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let gem = gem::new_user_memory_object( + device, + parent.as_ref(), + pages, + data.flags, + resv_obj, + commit, + )?; + drop(registered); + + data.handle = gem.create_handle(file)?; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_userptr size={:#x} handle={:#x?}\n", + file.inner().id, + size, + data.handle + ); + + Ok(0) + } + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. pub(crate) fn gem_mmap_offset( device: &AsahiDrmDevice, @@ -917,6 +1017,10 @@ impl File { } let gem = gem::Object::lookup_handle(file, data.handle)?; + // The caller maps a user-memory object's memory itself. + if gem.is_user_memory() { + return Err(EINVAL); + } data.offset = gem.create_mmap_offset()?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 854e3636e1d0c..3c3b667e61af5 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,6 +21,7 @@ use kernel::{ }, // }, error::Result, + mm::Mm, new_mutex, prelude::*, sync::aref::ARef, @@ -30,6 +31,7 @@ use kernel::{ use core::ops::Range; use core::sync::atomic::{ + AtomicI64, AtomicU64, AtomicUsize, Ordering, // @@ -63,6 +65,9 @@ pub(crate) struct AsahiObject { kernel_mappings: AtomicU64, /// Memory accounting and process attribution of the held backing. charge: memacct::Charge, + /// The pinned pages of a user-memory object (DRM_IOCTL_ASAHI_GEM_USERPTR), its backing. + /// The shmem layer unmaps them from the device before this is dropped. + user_memory: Option, /// Serializes overlapping timestamp writes through every binding of this BO. #[pin] pub(crate) timestamp_lock: Mutex<()>, @@ -113,9 +118,42 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { /// Takes the SG table of `gem`, which holds its pages for the device, and marks the /// object held (see [`set_mapping_held`]). +/// +/// For mappings the driver creates for itself or the firmware (kernel aliases, timestamp +/// buffers, firmware structures), which user-memory objects never back: those are refused. pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, +) -> Result> { + if gem.user_memory.is_some() { + return Err(EINVAL); + } + mapped_sg_table(gem, dev) +} + +/// Takes the SG table of `gem` for a user VM binding with `prot`, as [`held_sg_table`]. +/// +/// A read-only user-memory object takes only GPU read-only bindings (`EACCES` otherwise): its +/// pages may be another file's page cache, which the caller may not write. +pub(crate) fn held_user_sg_table<'a>( + gem: &'a Object, + dev: &'a kernel::device::Device, + prot: mmu::Prot, +) -> Result> { + if gem.user_memory.as_ref().is_some_and(|user| !user.writable) + && prot != mmu::PROT_GPU_SHARED_RO + { + return Err(EACCES); + } + mapped_sg_table(gem, dev) +} + +/// Takes the SG table of `gem` for a device mapping that [`held_sg_table`] or +/// [`held_user_sg_table`] already admitted (the caller holds a lease from one of them, or the +/// mapping exists). +pub(crate) fn mapped_sg_table<'a>( + gem: &'a Object, + dev: &'a kernel::device::Device, ) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); @@ -232,6 +270,7 @@ pub(crate) struct AsahiObjConfig { exportable: bool, kernel: bool, charge: memacct::Charge, + user_memory: Option, } /// Create a new kernel-owned GEM object. @@ -295,6 +334,7 @@ fn new_kernel_object_mapped( exportable: false, kernel: true, charge, + user_memory: None, }, )?; @@ -335,6 +375,7 @@ pub(crate) fn new_object( exportable: parent_object.is_none(), kernel: false, charge: memacct::Charge::user_current(commit), + user_memory: None, }, )?; @@ -342,6 +383,207 @@ pub(crate) fn new_object( Ok(gem) } +/// Create a user-memory object over `pages`, the caller's memory pinned by [`UserPages::pin`]. +/// +/// `commit` is the admission of the pages' size for the creating client. The pages count as held +/// by the GPU for the object's whole life, but are not attributed to the process again: they are +/// already part of its memory (anonymous memory or page cache it maps). The object cannot be +/// exported, CPU-mapped through the driver, vmapped, purged or idle-reclaimed. +pub(crate) fn new_user_memory_object( + dev: &AsahiDrmDevice, + parent: &kernel::device::Device, + pages: UserPages, + flags: u32, + parent_object: Option<&shmem::Object>, + commit: memacct::Commit, +) -> Result> { + if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() + { + return Err(EINVAL); + } + let writable = pages.writable; + // The page array is a heap allocation: moving `pages` into the object leaves it in place. + let array = core::ptr::slice_from_raw_parts(pages.pages.as_ptr(), pages.pages.len()); + // SAFETY: + // - `array` is the array of `pages`, which the object's data holds from its initialization + // on: the array stays allocated and its pages pinned until `pages` is dropped with that + // data. + // - Coherent access. The object is only ever mapped by user VM binds + // (`held_user_sg_table()`; `held_sg_table()` refuses it for every mapping the driver makes + // for itself or for the firmware), and only on the GPUs of `File::supports_user_memory()`, + // where a user VM bind is a set of GPU-only entries (`mmu::PROT_GPU_SHARED_*`) of the VM's + // page tables at the DMA addresses of the pages. The GPU reads and writes the pages + // through those entries and in no other way. This driver depends on such accesses being + // coherent with the CPU caches for its ordinary objects already: an object created with + // DRM_ASAHI_GEM_WRITEBACK is mapped cacheable into its process, which stores to it and + // loads from it while the GPU uses it through entries of the same kind, the driver can do + // no cache maintenance for those accesses, and the UAPI has no call for it. Whether the + // device is marked DMA-coherent does not enter into it: where it is not, the DMA layer + // would clean and invalidate CPU caches when it maps and unmaps, which is what the + // object's mapping skips. + let gem = unsafe { + shmem::Object::::new_external( + dev, + parent, + array, + writable, + shmem::ObjectConfig:: { + parent_resv_obj: parent_object, + ..Default::default() + }, + AsahiObjConfig { + flags, + exportable: false, + kernel: false, + charge: memacct::Charge::user(commit, None), + user_memory: Some(pages), + }, + ) + }?; + gem.charge.set_held(true); + mod_pr_debug!("AsahiObject new user-memory object: id={}\n", gem.id); + Ok(gem) +} + +/// Pages of the calling process pinned as the backing of a user-memory object, and counted in its +/// mm's `pinned_vm` against RLIMIT_MEMLOCK. +pub(crate) struct UserPages { + pages: KVVec<*mut kernel::bindings::page>, + writable: bool, + /// The mm whose `pinned_vm` counts the pages; it may outlive the address space. + mm: ARef, + /// Pages counted in `pinned_vm`: all of the range, including any not pinned yet. + counted: i64, +} + +// SAFETY: The pinned pages may be released from any thread; the mm reference is thread-safe. +unsafe impl Send for UserPages {} +// SAFETY: Shared access only reads the array. +unsafe impl Sync for UserPages {} + +/// Pages pinned per `pin_user_pages_fast()` call. +const PIN_CHUNK: usize = 16384; + +impl UserPages { + /// Pins the `size` bytes at `addr` in the calling process, for device writes as well if + /// `writable`. Both must be multiples of the page size, and `size` non-zero. + /// + /// All of the range is pinned, long-term, or none of it. Fails with `EFAULT` when a page cannot + /// be pinned: unmapped, a device or PFN mapping, or a writable pin of a mapping that is not + /// writable or whose filesystem tracks writes (`FOLL_LONGTERM` refuses those). Fails with + /// `ENOMEM` when the pages would take the process past RLIMIT_MEMLOCK without CAP_IPC_LOCK, + /// counting every page pinned this way or by other long-term pinners. + /// + /// Anonymous pages pinned here stay with this process across fork(): the child gets copies. + /// Pages unmapped by the process stay pinned, and backing the object, until it is freed. + pub(crate) fn pin(addr: usize, size: usize, writable: bool) -> Result { + let page_size = kernel::page::PAGE_SIZE; + if size == 0 || (addr | size) & (page_size - 1) != 0 { + return Err(EINVAL); + } + addr.checked_add(size).ok_or(EFAULT)?; + let count = size / page_size; + let mm: ARef = kernel::current!() + .mm() + .map(|mm| ARef::from(&**mm)) + .ok_or(EFAULT)?; + let pinned_vm = Self::pinned_vm(&mm); + let added = i64::try_from(count).map_err(|_| ENOMEM)?; + let total = pinned_vm + .fetch_add(added, Ordering::AcqRel) + .saturating_add(added); + if !memlock_allows(total, memlock_limit_pages()) { + // SAFETY: `capable()` has no preconditions. + if !unsafe { kernel::bindings::capable(kernel::bindings::CAP_IPC_LOCK as i32) } { + pinned_vm.fetch_sub(added, Ordering::AcqRel); + return Err(ENOMEM); + } + } + // From here on, dropping `this` unpins what it holds and uncounts all `count` pages. + let mut this = Self { + pages: KVVec::new(), + writable, + mm, + counted: added, + }; + this.pages.reserve(count, GFP_KERNEL)?; + let mut flags = kernel::bindings::FOLL_LONGTERM; + if writable { + flags |= kernel::bindings::FOLL_WRITE; + } + while this.pages.len() < count { + let done = this.pages.len(); + let chunk = (count - done).min(PIN_CHUNK); + // SAFETY: The array has room for `count` entries, of which `done` are in use; the + // call fills at most `chunk` more. + let ret = unsafe { + kernel::bindings::pin_user_pages_fast( + (addr + done * page_size) as _, + chunk as _, + flags, + this.pages.as_mut_ptr().add(done), + ) + }; + if ret <= 0 { + return Err(if ret < 0 { + Error::from_errno(ret) + } else { + EFAULT + }); + } + let ret = (ret as usize).min(chunk); + // SAFETY: The call stored `ret` valid page pointers after the first `done`. + unsafe { this.pages.inc_len(ret) }; + } + Ok(this) + } + + fn pinned_vm(mm: &Mm) -> &AtomicI64 { + // SAFETY: `pinned_vm` is an aligned atomic64 of the live mm, which the kernel only changes + // atomically. + unsafe { + AtomicI64::from_ptr(core::ptr::addr_of_mut!( + (*mm.as_raw()).__bindgen_anon_1.pinned_vm.counter + )) + } + } +} + +impl Drop for UserPages { + fn drop(&mut self) { + if !self.pages.is_empty() { + // SAFETY: The array holds pages pinned by `pin_user_pages_fast()` and not yet + // unpinned. Pages the device may have written are marked dirty. + unsafe { + kernel::bindings::unpin_user_pages_dirty_lock( + self.pages.as_mut_ptr(), + self.pages.len() as _, + self.writable, + ) + }; + } + Self::pinned_vm(&self.mm).fetch_sub(self.counted, Ordering::AcqRel); + } +} + +/// RLIMIT_MEMLOCK of the calling process, in pages. +fn memlock_limit_pages() -> u64 { + let task = kernel::current!().as_ptr(); + // SAFETY: The current task and its signal structure are live; the limit is read once, as + // `rlimit()` does. + let bytes = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!( + (*(*task).signal).rlim[kernel::bindings::RLIMIT_MEMLOCK as usize].rlim_cur + )) + }; + (bytes >> kernel::page::PAGE_SHIFT) as u64 +} + +/// Whether `pinned` pages stay within a memlock limit of `limit` pages. +fn memlock_allows(pinned: i64, limit: u64) -> bool { + u64::try_from(pinned).is_ok_and(|pinned| pinned <= limit) +} + #[vtable] impl DriverObject for AsahiObject { type Driver = AsahiDrmDriver; @@ -361,6 +603,7 @@ impl DriverObject for AsahiObject { kernel: args.kernel, kernel_mappings: AtomicU64::new(0), charge: args.charge, + user_memory: args.user_memory, timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } @@ -376,6 +619,7 @@ impl DriverObject for AsahiObject { exportable: true, kernel: false, charge: memacct::Charge::none(), + user_memory: None, }) } @@ -442,7 +686,15 @@ impl AsahiObject { } pub(crate) fn idle_reclaim_candidate(&self) -> bool { - !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 + !self.kernel + && !self.exportable + && self.user_memory.is_none() + && self.kernel_mappings.load(Ordering::Acquire) == 0 + } + + /// Whether this object is backed by the creating process's own memory. + pub(crate) fn is_user_memory(&self) -> bool { + self.user_memory.is_some() } } @@ -466,7 +718,10 @@ pub(crate) fn prepare_new_object( /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 || gem.exportable { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 + || gem.exportable + || gem.user_memory.is_some() + { return Err(EINVAL); } Ok(u32::from(gem.madvise(i32::from(dontneed)))) @@ -493,6 +748,158 @@ mod allocation_policy_tests { } } +#[kernel::macros::kunit_tests(asahi_gem_user_memory)] +mod user_memory_tests { + use super::*; + use kernel::{ + bindings, + page::PAGE_SIZE, + uaccess::{UserPtr, UserSlice}, + }; + + const PROT_READ: u64 = 1; + const PROT_WRITE: u64 = 2; + const MAP_PRIVATE: u64 = 2; + const MAP_ANONYMOUS: u64 = 0x20; + + /// Maps `len` bytes of anonymous memory into the test's user address space. + fn map(len: usize, prot: u64) -> Result { + // SAFETY: Called on a KUnit test thread. KUnit attaches a user mm for the test and + // removes the mapping with it when the test ends. + let addr = unsafe { + bindings::kunit_vm_mmap( + bindings::kunit_get_current_test(), + core::ptr::null_mut(), + 0, + len as _, + prot as _, + (MAP_PRIVATE | MAP_ANONYMOUS) as _, + 0, + ) + } as usize; + if addr == 0 || addr > usize::MAX - 4095 { + return Err(ENOMEM); + } + Ok(addr) + } + + fn pinned_vm() -> Result { + let mm: ARef = kernel::current!() + .mm() + .map(|mm| ARef::from(&**mm)) + .ok_or(EINVAL)?; + Ok(UserPages::pinned_vm(&mm).load(Ordering::Acquire)) + } + + fn read_pinned(pages: &UserPages, index: usize, offset: usize) -> Result { + let page = *pages.pages.get(index).ok_or(EINVAL)?; + // SAFETY: The page is pinned while `pages` lives; the word is inside it. + Ok(unsafe { + let p = bindings::kmap_local_page(page); + let value = core::ptr::read_volatile(p.cast::().add(offset).cast::()); + bindings::kunmap_local(p); + value + }) + } + + #[test] + fn memlock_limit_policy() { + assert!(memlock_allows(0, 0)); + assert!(memlock_allows(512, 512)); + assert!(!memlock_allows(513, 512)); + assert!(!memlock_allows(-1, 512)); + assert!(memlock_allows(i64::MAX, u64::MAX)); + } + + #[test] + fn pins_anonymous_memory() -> Result { + let addr = map(4 * PAGE_SIZE, PROT_READ | PROT_WRITE)?; + let before = pinned_vm()?; + let pages = UserPages::pin(addr, 4 * PAGE_SIZE, true)?; + assert_eq!(pages.pages.len(), 4); + assert_eq!(pinned_vm()?, before + 4); + // CPU writes through the caller's mapping land in the pinned pages. + UserSlice::new(UserPtr::from_addr(addr + PAGE_SIZE + 8), 4) + .writer() + .write_slice(&0x1234_5678u32.to_ne_bytes())?; + assert_eq!(read_pinned(&pages, 1, 8)?, 0x1234_5678); + drop(pages); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn bad_ranges_pin_nothing() -> Result { + let addr = map(3 * PAGE_SIZE, PROT_READ | PROT_WRITE)?; + // SAFETY: Unmaps the last page of the test's own mapping. + assert_eq!( + unsafe { bindings::vm_munmap((addr + 2 * PAGE_SIZE) as _, PAGE_SIZE) }, + 0 + ); + let before = pinned_vm()?; + assert_eq!( + UserPages::pin(addr + 1, PAGE_SIZE, true).err(), + Some(EINVAL) + ); + assert_eq!(UserPages::pin(addr, 0, true).err(), Some(EINVAL)); + assert_eq!( + UserPages::pin(addr, PAGE_SIZE + 1, true).err(), + Some(EINVAL) + ); + // Two pages pin before the hole: those are unpinned again. + assert_eq!( + UserPages::pin(addr, 3 * PAGE_SIZE, true).err(), + Some(EFAULT) + ); + assert_eq!( + UserPages::pin(addr + 2 * PAGE_SIZE, PAGE_SIZE, false).err(), + Some(EFAULT) + ); + let top = usize::MAX & !(PAGE_SIZE - 1); + assert_eq!( + UserPages::pin(top, 2 * PAGE_SIZE, false).err(), + Some(EFAULT) + ); + assert_eq!(pinned_vm()?, before); + // What remains of the mapping still pins. + drop(UserPages::pin(addr, 2 * PAGE_SIZE, true)?); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn read_only_mappings_pin_read_only() -> Result { + let addr = map(2 * PAGE_SIZE, PROT_READ)?; + let before = pinned_vm()?; + assert_eq!( + UserPages::pin(addr, 2 * PAGE_SIZE, true).err(), + Some(EFAULT) + ); + assert_eq!(pinned_vm()?, before); + let pages = UserPages::pin(addr, 2 * PAGE_SIZE, false)?; + assert_eq!(pages.pages.len(), 2); + assert!(!pages.writable); + assert_eq!(pinned_vm()?, before + 2); + drop(pages); + assert_eq!(pinned_vm()?, before); + Ok(()) + } + + #[test] + fn pages_outlive_the_mapping() -> Result { + let addr = map(PAGE_SIZE, PROT_READ | PROT_WRITE)?; + UserSlice::new(UserPtr::from_addr(addr + 16), 4) + .writer() + .write_slice(&0xa5a5_5a5au32.to_ne_bytes())?; + let pages = UserPages::pin(addr, PAGE_SIZE, true)?; + // SAFETY: Unmaps the test's own mapping; the pinned page stays allocated. + assert_eq!(unsafe { bindings::vm_munmap(addr as _, PAGE_SIZE) }, 0); + assert_eq!(read_pinned(&pages, 0, 16)?, 0xa5a5_5a5a); + drop(pages); + Ok(()) + } +} + /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ad7f6273141ed..3ac6e589a6393 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2382,8 +2382,9 @@ impl HandoffFlush { } } +/// Copies the DMA segments of `gem`, whose device mapping a caller's lease already admitted. fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { - let sgt = crate::gem::held_sg_table(gem, dev)?; + let sgt = crate::gem::mapped_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { @@ -2996,7 +2997,7 @@ impl Vm { self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let _bind_lease = crate::gem::held_user_sg_table(gem, dev, prot)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index a792cdd1bec08..9748357561077 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -89,7 +89,7 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let sg_lease = crate::gem::held_user_sg_table(gem, dev, prot)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 3a0bf7ae28b58..c51b239fb4917 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -96,6 +96,12 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ DRM_ASAHI_GEM_MADVISE, + + /** + * @DRM_ASAHI_GEM_USERPTR: Create a buffer object over the caller's own + * memory (DRM_ASAHI_FEATURE_USERPTR). + */ + DRM_ASAHI_GEM_USERPTR, }; #define DRM_ASAHI_MAX_CLUSTERS 64 @@ -489,6 +495,12 @@ enum drm_asahi_feature { */ #define DRM_ASAHI_FEATURE_COMPUTE_VISIBLE_COMPLETION ((1ULL) << 34) +/* + * DRM_ASAHI_FEATURE_USERPTR: DRM_IOCTL_ASAHI_GEM_USERPTR is available. A + * feature bit like those of enum drm_asahi_feature, defined here like bit 32. + */ +#define DRM_ASAHI_FEATURE_USERPTR ((1ULL) << 37) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -671,6 +683,12 @@ enum drm_asahi_gem_flags { * @DRM_ASAHI_GEM_VM_PRIVATE: BO is private to this GPU VM (no exports). */ DRM_ASAHI_GEM_VM_PRIVATE = (1L << 1), + + /** + * @DRM_ASAHI_GEM_USERPTR_READ_ONLY: The GPU only reads the memory of a + * DRM_IOCTL_ASAHI_GEM_USERPTR object. Only that ioctl accepts it. + */ + DRM_ASAHI_GEM_USERPTR_READ_ONLY = (1L << 2), }; /** @@ -695,6 +713,75 @@ struct drm_asahi_gem_create { __u32 pad; }; +/** + * struct drm_asahi_gem_userptr - Arguments passed to + * DRM_IOCTL_ASAHI_GEM_USERPTR + * + * Creates a buffer object whose backing is the caller's memory at + * [@userptr, @userptr + @size), without a copy: the CPU accesses it through + * the caller's own mapping, and the GPU through VM_BIND mappings of the + * object, as for any other object. Available when DRM_ASAHI_FEATURE_USERPTR + * is set; otherwise the ioctl fails with EOPNOTSUPP or EINVAL. + * + * The pages of the range are pinned when the object is created and stay + * pinned until it is freed: all of them or, on failure, none. They are the + * pages the range maps at creation; later changes to the caller's mappings + * (munmap, mremap, a new mapping over the range, truncating a mapped file) + * leave the object on those pages, which stay allocated until it is freed. + * After fork(), the child gets copies of the pinned anonymous pages; the + * object stays the parent's memory. The CPU and GPU views are coherent + * without cache maintenance. + * + * The pinned pages count against the caller's RLIMIT_MEMLOCK, together with + * its other long-term pins, unless it has CAP_IPC_LOCK (ENOMEM otherwise), + * and their size against the client and device limits of + * drm_asahi_params_global::client_memory_limit and + * drm_asahi_params_global::memory_limit (ENOMEM). They are never released by + * memory-pressure reclaim. + * + * Without DRM_ASAHI_GEM_USERPTR_READ_ONLY, the whole range must be mapped + * writable, and not be a shared mapping of a file whose filesystem tracks + * writes. With it, any readable mapping is accepted, and the object takes + * only GPU read-only VM_BIND mappings (a mapping with DRM_ASAHI_BIND_WRITE + * fails with EACCES). Pages of private mappings, file mappings included, are + * pinned as the caller's own copies, as a write would make them: a private + * file mapping is copied out of the page cache. A read-only shared file + * mapping pins the page-cache pages themselves, which then stay resident + * while the object lives. Device and PFN mappings, including CPU mappings of + * other buffer objects, are refused (EFAULT). + * + * The object cannot be mapped with DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, exported, + * given advice with DRM_IOCTL_ASAHI_GEM_MADVISE, bound with + * DRM_IOCTL_ASAHI_GEM_BIND_OBJECT or registered as a VM status mirror (all + * EINVAL). + */ +struct drm_asahi_gem_userptr { + /** + * @userptr: Start of the range, aligned to both the CPU page size and + * the GPU page size (16 KiB). + */ + __u64 userptr; + + /** @size: Size of the range, a non-zero multiple of both page sizes. */ + __u64 size; + + /** + * @flags: DRM_ASAHI_GEM_VM_PRIVATE and DRM_ASAHI_GEM_USERPTR_READ_ONLY. + */ + __u32 flags; + + /** + * @vm_id: VM ID to assign to the BO, if DRM_ASAHI_GEM_VM_PRIVATE is set + */ + __u32 vm_id; + + /** @handle: Returned GEM handle for the BO */ + __u32 handle; + + /** @pad: MBZ */ + __u32 pad; +}; + /** * struct drm_asahi_gem_mmap_offset - Arguments passed to * DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET @@ -2193,6 +2280,7 @@ enum { DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOCTL_ASAHI(W, QUEUE_DESTROY, queue_destroy), DRM_IOCTL_ASAHI_SUBMIT = DRM_IOCTL_ASAHI(W, SUBMIT, submit), DRM_IOCTL_ASAHI_GEM_MADVISE = DRM_IOCTL_ASAHI(WR, GEM_MADVISE, gem_madvise), + DRM_IOCTL_ASAHI_GEM_USERPTR = DRM_IOCTL_ASAHI(WR, GEM_USERPTR, gem_userptr), }; #if defined(__cplusplus) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index acabd105e3b1f..01e0e6cfbced8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -233,6 +233,15 @@ const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; #endif +#if IS_ENABLED(CONFIG_KUNIT) +/* + * struct bus_dma_region, for KUnit tests that give a test device a + * dma_range_map. Not for drivers or abstractions: the header is the DMA + * layer's own. + */ +#include +#endif + #if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) #include "../../drivers/android/binder/rust_binder.h" #include "../../drivers/android/binder/rust_binder_events.h" diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fa77f4d3d9f55..aa6b5e17d603d 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -130,6 +130,10 @@ pub struct Object { sgt_lock: Mutex<()>, /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, + /// The device mapping of an object over driver-supplied pages ([`Self::new_external`]). + /// Declared before `inner`, which holds those pages, so that it is unmapped first, and + /// initialized after it, so that nothing is mapped before `inner` holds them. + external: Option>, #[pin] inner: T, } @@ -172,6 +176,30 @@ impl Object { evict: None, }; + /// `drm_gem_object_funcs` vtable for objects over driver-supplied pages. + /// + /// They have no shmem pages for the C helpers to map, pin or export: every CPU, kernel and + /// dma-buf view through the DRM core fails instead (no `mmap` or `vm_ops`, no `vmap`, `pin` + /// or `get_sg_table`, and an `export` that refuses, since without one the core would use its + /// generic export). + const EXTERNAL_VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: None, + export: Some(Self::refuse_export_callback), + pin: None, + unpin: None, + get_sg_table: None, + vmap: None, + vunmap: None, + mmap: None, + status: None, + rss: None, + vm_ops: ptr::null(), + evict: None, + }; + /// Return a raw pointer to the embedded drm_gem_shmem_object. fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { self.obj.get() @@ -213,6 +241,13 @@ impl Object { let _ = unsafe { KBox::from_raw(this) }; } + extern "C" fn refuse_export_callback( + _obj: *mut bindings::drm_gem_object, + _flags: i32, + ) -> *mut bindings::dma_buf { + EINVAL.to_ptr() + } + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> where @@ -222,6 +257,10 @@ impl Object { if self.size() < SIZE { return Err(ENOSPC); } + // The C helper would allocate shmem pages for an object over driver-supplied pages. + if self.is_external() { + return Err(EINVAL); + } let mut map: MaybeUninit = MaybeUninit::uninit(); let guard = DmaResvGuard::new(self); @@ -303,7 +342,10 @@ impl Object { let lease = SGLease { obj: self.into() }; // SAFETY: Import attachment ownership is immutable after publication. - let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + let sgt = if let Some(external) = &self.external { + // Mapped at creation and never rebuilt: once the device is unbound, there is none. + external.access(dev)?.sgt.get() + } else if !unsafe { (*self.as_raw()).import_attach }.is_null() { // Imports belong to the exporter. Never register them for Rust devres cleanup, // rebuild their tables, or release their backing. // SAFETY: The attachment table stays valid until this object is freed. @@ -351,6 +393,7 @@ impl Object { sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), + external: None, inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -388,6 +431,123 @@ impl Object { Ok(obj) } + /// Create a DRM object whose backing is `pages`, which the driver data built from `args` + /// holds, instead of shmem pages. Its size is that of the pages. + /// + /// The object has no shmem file. Its device SG table is built and DMA-mapped here, once, for + /// `parent` (the DRM device's parent), and is unmapped when the object is freed or the device + /// is unbound. The mapping does no CPU cache maintenance, on map or unmap: the pages are the + /// owner's to read and write through cacheable CPU mappings, and invalidating CPU caches when + /// the object is freed would discard data written through them. The device must therefore be + /// coherent with CPU caches for this memory. `writable` selects a bidirectional mapping; + /// otherwise the mapping is for device reads only. + /// + /// The device must reach the pages themselves at the DMA addresses of the table: a bounce + /// buffer, which a mapping without CPU synchronization never updates, will not do. The pages + /// are therefore mapped with `DMA_ATTR_NO_BOUNCE`, and the DMA layer fails the mapping where + /// it would substitute other memory for a page: with `EIO` for pages it would bounce (the + /// device cannot address them, bouncing is forced, or the device needs unencrypted memory), + /// and with `EOPNOTSUPP` for a device that is mapped by DMA operations of its own. `EIO` is + /// the DMA layer's error for any page it cannot map, a refused bounce buffer or not. + /// + /// The driver data is initialized first, and the pages are mapped only once it holds them: if + /// `T::new()` or its initializer fails, nothing was mapped, and if the mapping fails, the + /// driver data is dropped with no mapping left. When the object is freed, the mapping is + /// removed before the driver data is dropped. + /// + /// The object cannot be CPU-mapped or exported through the DRM core, kernel-mapped with + /// [`Self::vmap`], released by idle reclaim or purged: those fail, or report that the backing + /// is not releasable. + /// + /// # Safety + /// + /// - From the moment the initializer `T::new()` returns for `args` has succeeded, and until + /// the driver data it built is dropped, `pages` points to an array of valid pages, and the + /// array and the pages stay allocated and stay that data's backing. `pages` is not + /// dereferenced before that, nor if the initializer fails, so the array may be memory that + /// `args` owns. + /// - The device accesses the pages coherently with the CPU caches, through every mapping the + /// driver gives it of them at the DMA addresses of the table and for as long as the object + /// exists: its next read of a location sees a store made through a cacheable CPU mapping, + /// and the CPU's next load sees a store of the device, with no cache maintenance in + /// between. Nothing here establishes or checks that. `DMA_ATTR_NO_BOUNCE` makes the DMA + /// addresses those of the pages themselves and `DMA_ATTR_SKIP_CPU_SYNC` leaves the caches + /// alone, for a device that is not coherent as for one that is, and whether or not the + /// device is marked DMA-coherent: the mapping succeeds either way. The caller has to know + /// it of its hardware. + pub unsafe fn new_external( + dev: &Device, + parent: &device::Device, + pages: *const [*mut bindings::page], + writable: bool, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + if parent.as_raw() != dev.as_ref().as_ref().as_raw() { + return Err(EINVAL); + } + if config.map_wc || config.reclaimable_cpu_mappings || config.pages_bounded_reclaim { + return Err(EINVAL); + } + if pages.is_empty() { + return Err(EINVAL); + } + let size = pages + .len() + .checked_mul(crate::page::PAGE_SIZE) + .ok_or(EOVERFLOW)?; + + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + sgt_access: AtomicUsize::new(0), + // The driver data takes the pages over, and an initializer that fails may already + // have released them: map them only once it has succeeded. A failure after that + // drops the mapping, if it was made, and then the driver data. + inner <- T::new(dev, size, args), + external: Some(Devres::new( + parent, + // SAFETY: `inner` is initialized, so per the caller's contract `pages` is an + // array of valid pages that stay allocated until `inner` is dropped, which + // comes after this mapping is. The caller answers for the device's coherent + // access to them. + Ok::<_, Error>(unsafe { ExternalDma::new(parent, &*pages, size, writable) }?), + )?), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::EXTERNAL_VTABLE }; + + // A private object has no shmem file, so that no helper can ever allocate pages for it. + // SAFETY: The arguments are all valid via the type invariants; the object is not + // published yet. + unsafe { bindings::drm_gem_private_object_init(dev.as_raw(), new.as_raw(), size) }; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_private_object_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + Ok(obj) + } + + /// Whether the object's backing is driver-supplied pages ([`Self::new_external`]). + pub fn is_external(&self) -> bool { + self.external.is_some() + } + /// `gem_create_object` callback, for the objects the DRM core allocates on the driver's /// behalf (dma-buf imports and dumb buffers) before it initializes their shmem part. /// @@ -409,6 +569,7 @@ impl Object { sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), + external: None, inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -459,7 +620,8 @@ impl Object { /// /// The caller holds the object's DMA reservation lock. pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 1 && + !self.is_external() && + self.sgt_access.load(Ordering::Acquire) == 1 && // SAFETY: The caller establishes the reservation-lock contract. unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } @@ -482,7 +644,8 @@ impl Object { /// /// The caller holds the object's DMA reservation lock. pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 0 && + !self.is_external() && + self.sgt_access.load(Ordering::Acquire) == 0 && // SAFETY: The caller establishes the reservation-lock contract. unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } @@ -498,6 +661,9 @@ impl Object { /// The caller holds this object's DMA reservation lock, has removed and invalidated every /// device mapping, and excludes new device access. pub unsafe fn release_idle_pages_locked(&self) -> Result { + if self.is_external() { + return Err(EBUSY); + } self.sgt_access .compare_exchange(0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| EBUSY)?; @@ -511,8 +677,11 @@ impl Object { /// Apply purge advice (0 = needed, >0 = may be purged) under the /// object's reservation lock. Returns false if the object was already - /// purged; the advice is then ignored. + /// purged; the advice is then ignored. Objects over driver-supplied pages take no advice. pub fn madvise(&self, madv: i32) -> bool { + if self.is_external() { + return true; + } let _guard = DmaResvGuard::new(self); // SAFETY: The reservation lock is held, as the C helper requires. unsafe { bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0 } @@ -536,6 +705,9 @@ impl Object { /// The caller holds this object's DMA reservation lock, has removed and /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { + if self.is_external() { + return Err(EBUSY); + } // SAFETY: The caller's contract; the exclusive state below excludes new leases. if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } @@ -807,6 +979,110 @@ unsafe impl Send for SGTableCleanup {} // SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. unsafe impl Sync for SGTableCleanup {} +/// The device mapping of an object over driver-supplied pages: an SG table over the pages, +/// DMA-mapped without CPU cache maintenance and without bounce buffers. +/// +/// # Invariants +/// +/// `sgt` is an initialized table, DMA-mapped for `dev` with `dir`, over pages that stay valid +/// while this exists. +struct ExternalDma { + sgt: KBox>, + dev: ARef, + dir: bindings::dma_data_direction, +} + +/// The number of live [`ExternalDma`] mappings, for the tests of the order in which a mapping and +/// the owner of its pages go. +#[cfg(CONFIG_KUNIT = "y")] +static EXTERNAL_MAPPINGS: AtomicUsize = AtomicUsize::new(0); + +// SAFETY: The table is only read once built, and the device reference may move between threads. +unsafe impl Send for ExternalDma {} +// SAFETY: See above. +unsafe impl Sync for ExternalDma {} + +impl ExternalDma { + /// The attributes of the mapping. No CPU cache maintenance, on map or unmap: the pages are + /// their owner's to use through cacheable CPU mappings. And no bounce buffer, which such a + /// mapping would never update: the DMA layer fails the mapping instead of substituting other + /// memory for a page. + const ATTRS: c_ulong = + (bindings::DMA_ATTR_SKIP_CPU_SYNC | bindings::DMA_ATTR_NO_BOUNCE) as c_ulong; + + /// # Safety + /// + /// - `pages` are valid pages that stay allocated for the life of the returned mapping. + /// - `dev` accesses them coherently with the CPU caches while the mapping exists, as + /// [`Object::new_external`] requires of its caller: the attributes of this mapping keep + /// the DMA layer from substituting or synchronizing memory, and make no device coherent. + unsafe fn new( + dev: &device::Device, + pages: &[*mut bindings::page], + size: usize, + writable: bool, + ) -> Result { + if pages.is_empty() { + return Err(EINVAL); + } + let sgt = KBox::new(Opaque::::zeroed(), GFP_KERNEL)?; + // SAFETY: `dev` is a valid device. + let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { + 0 => u32::MAX, + max => u32::try_from(max).unwrap_or(u32::MAX), + }; + // SAFETY: `sgt` is zeroed memory for a table, `pages` holds at least one valid page and + // covers `size` bytes. + to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + // The table only reads the array. + pages.as_ptr().cast_mut(), + pages.len().try_into()?, + 0, + size, + max_segment, + GFP_KERNEL.as_raw(), + ) + })?; + let dir = if writable { + bindings::dma_data_direction_DMA_BIDIRECTIONAL + } else { + bindings::dma_data_direction_DMA_TO_DEVICE + }; + // SAFETY: The table was built above; `dev` is bound. + let mapped = to_result(unsafe { + bindings::dma_map_sgtable(dev.as_raw(), sgt.get(), dir, Self::ATTRS) + }); + if let Err(e) = mapped { + // SAFETY: The table was allocated above and is not mapped. + unsafe { bindings::sg_free_table(sgt.get()) }; + return Err(e); + } + #[cfg(CONFIG_KUNIT = "y")] + EXTERNAL_MAPPINGS.fetch_add(1, Ordering::AcqRel); + // INVARIANT: The table is built and mapped. + Ok(Self { + sgt, + dev: dev.into(), + dir, + }) + } +} + +impl Drop for ExternalDma { + fn drop(&mut self) { + // SAFETY: Per the type invariants the table is mapped for `dev` with `dir`; it is unmapped + // and freed once, here. + unsafe { + bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.get(), self.dir, Self::ATTRS); + bindings::sg_free_table(self.sgt.get()); + } + #[cfg(CONFIG_KUNIT = "y")] + EXTERNAL_MAPPINGS.fetch_sub(1, Ordering::AcqRel); + } +} + #[kunit_tests(rust_drm_gem_shmem)] mod tests { use super::*; @@ -1154,6 +1430,543 @@ mod tests { Ok(()) } + fn page_pointers( + pages: &[crate::types::Owned; N], + ) -> [*mut bindings::page; N] { + core::array::from_fn(|i| pages[i].as_ptr()) + } + + fn alloc_pages() -> Result<[crate::types::Owned; N]> { + let mut pages = KVec::with_capacity(N, GFP_KERNEL)?; + for _ in 0..N { + pages.push( + crate::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?, + GFP_KERNEL, + )?; + } + pages.try_into().map_err(|_| EINVAL) + } + + #[test] + fn external_pages_back_the_table() -> Result { + let (dev, drm) = create_drm_dev()?; + // Declared before the object, so they outlive it. + let pages = alloc_pages::<3>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }?; + assert!(obj.is_external()); + assert_eq!(obj.size(), 3 * PAGE_SIZE); + // SAFETY: The object is live; nothing else changes these fields. + let (filp, shmem_pages) = unsafe { ((*obj.as_raw()).filp, (*obj.as_raw_shmem()).pages) }; + assert!(filp.is_null()); + assert!(shmem_pages.is_null()); + let table = obj.sg_table(dev.as_ref().as_ref())?; + // The faux device maps directly: the DMA addresses are the pages' physical addresses. + let mut next = raw.iter(); + let mut total = 0; + for segment in table.iter() { + let mut offset = 0; + while offset < u64::from(segment.dma_len()) { + let page = *next.next().ok_or(EINVAL)?; + // SAFETY: `page` is one of the live test pages. + assert_eq!( + unsafe { bindings::page_to_phys(page) }, + segment.dma_address() + offset + ); + offset += PAGE_SIZE as u64; + } + total += segment.dma_len() as usize; + } + assert_eq!(total, 3 * PAGE_SIZE); + assert!(next.next().is_none()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(table); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + // The pages are the caller's: freeing the object leaves their contents alone. + let marker = 0x5au8; + // SAFETY: The page is live and the byte is within it. + unsafe { pages[1].write_raw(&marker, 7, 1)? }; + drop(obj); + let mut back = 0u8; + // SAFETY: As above. + unsafe { pages[1].read_raw(&mut back, 7, 1)? }; + assert_eq!(back, marker); + Ok(()) + } + + #[test] + fn external_objects_refuse_cpu_views_and_reclaim() -> Result { + let (dev, drm) = create_drm_dev()?; + let pages = alloc_pages::<2>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + false, + ObjectConfig::default(), + (), + ) + }?; + assert_eq!(obj.vmap::<4>().err(), Some(EINVAL)); + let mut map = MaybeUninit::::zeroed(); + // SAFETY: A live object; the core fails without a vmap callback. + assert_eq!( + unsafe { bindings::drm_gem_vmap(obj.as_raw(), map.as_mut_ptr()) }, + -(bindings::EOPNOTSUPP as i32) + ); + // SAFETY: The vtable is static. + let funcs = unsafe { &*(*obj.as_raw()).funcs }; + assert!(funcs.mmap.is_none() && funcs.vm_ops.is_null()); + assert!(funcs.pin.is_none() && funcs.get_sg_table.is_none()); + let export = funcs.export.ok_or(EINVAL)?; + // SAFETY: The callback refuses without touching the object. + let buf = unsafe { export(obj.as_raw(), 0) }; + assert_eq!(buf as isize, -(bindings::EINVAL as isize)); + // Advice is ignored, and leases or none, the backing is never releasable. + assert!(obj.madvise(1)); + let lease = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device mappings exist in this test. + let (madv, releasable, released) = unsafe { + ( + obj.madv_locked(), + obj.sole_sg_lease_releasable_locked(), + obj.release_idle_pages_locked(), + ) + }; + assert_eq!(madv, 0); + assert!(!releasable); + assert_eq!(released, Err(EBUSY)); + } + drop(lease); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: As above, with no lease left. + let (releasable, released, purged) = unsafe { + ( + obj.unleased_pages_releasable_locked(), + obj.release_idle_pages_locked(), + obj.purge_locked(), + ) + }; + assert!(!releasable); + assert_eq!(released, Err(EBUSY)); + assert_eq!(purged, Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn external_objects_need_pages_and_their_device() -> Result { + let (dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"OtherKunit", None)?; + let pages = alloc_pages::<1>()?; + let raw = page_pointers(&pages); + let no_pages: [*mut bindings::page; 0] = []; + // SAFETY: No pages are referenced. + let empty = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &no_pages[..], + true, + ObjectConfig::default(), + (), + ) + }; + assert_eq!(empty.err(), Some(EINVAL)); + // SAFETY: `pages` outlive the attempt. + let wrong = unsafe { + Object::::new_external( + &drm, + reg.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }; + assert_eq!(wrong.err(), Some(EINVAL)); + // SAFETY: As above. + let wc = unsafe { + Object::::new_external( + &drm, + dev.as_ref().as_ref(), + &raw[..], + true, + ObjectConfig { + map_wc: true, + ..Default::default() + }, + (), + ) + }; + assert_eq!(wc.err(), Some(EINVAL)); + Ok(()) + } + + /// The entry that ends a device's `dma_range_map`, the map that tells how the device's bus sees + /// CPU addresses (firmware describes it for a real device). + const BUS_MAP_END: bindings::bus_dma_region = bindings::bus_dma_region { + cpu_start: 0, + dma_start: 0, + size: 0, + }; + + /// Puts `dev` on the bus that `map`, which ends with [`BUS_MAP_END`], describes. + /// + /// # Safety + /// + /// `dev` is private to the calling test, and has no bus map and no DMA mapping yet. + unsafe fn set_bus_map( + dev: &device::Device, + map: [bindings::bus_dma_region; N], + ) -> Result { + let map = KBox::new(map, GFP_KERNEL)?; + // SAFETY: Per the safety requirements nothing else uses this field. The driver core + // frees the map with kfree() when the device goes. + unsafe { + (*dev.as_raw()).dma_range_map = KBox::into_raw(map).cast::() + }; + Ok(()) + } + + #[test] + fn external_mapping_follows_a_bus_offset() -> Result { + // The bus of this device sees every CPU address 1 TiB up. + const OFFSET: u64 = 1 << 40; + let (dev, drm) = create_drm_dev()?; + let parent: &device::Device = dev.as_ref().as_ref(); + // SAFETY: The faux device is private to this test and has no mapping yet. + unsafe { + set_bus_map( + parent, + [ + bindings::bus_dma_region { + cpu_start: 0, + dma_start: OFFSET, + size: OFFSET, + }, + BUS_MAP_END, + ], + )? + }; + let pages = alloc_pages::<3>()?; + let raw = page_pointers(&pages); + // SAFETY: `pages` outlive the object. + let obj = unsafe { + Object::::new_external( + &drm, + parent, + &raw[..], + true, + ObjectConfig::default(), + (), + ) + }?; + // The DMA layer applied the offset: no DMA address is its page's physical address. The + // device reaches the pages themselves all the same, and the object was accepted. + let table = obj.sg_table(parent)?; + let mut next = raw.iter(); + for segment in table.iter() { + let mut offset = 0; + while offset < u64::from(segment.dma_len()) { + let page = *next.next().ok_or(EINVAL)?; + // SAFETY: `page` is one of the live test pages. + assert_eq!( + unsafe { bindings::page_to_phys(page) } + OFFSET, + segment.dma_address() + offset + ); + offset += PAGE_SIZE as u64; + } + } + assert!(next.next().is_none()); + Ok(()) + } + + // A second fake driver, whose object data owns the pages the object is created over (as a + // driver that pins memory for an object does) and whose initializer can fail after taking + // them. + + /// The number of device mappings of external objects that existed when a [`PageOwner`] last + /// let go of its pages. + static MAPPED_AT_RELEASE: AtomicUsize = AtomicUsize::new(usize::MAX); + + struct PageOwner { + _pages: [crate::types::Owned; 2], + fail: bool, + } + + impl Drop for PageOwner { + fn drop(&mut self) { + MAPPED_AT_RELEASE.store(EXTERNAL_MAPPINGS.load(Ordering::Acquire), Ordering::Release); + } + } + + struct OwnerDriver; + struct OwnerFile; + #[pin_data] + struct OwnerObject { + _owner: PageOwner, + } + + impl drm::file::DriverFile for OwnerFile { + type Driver = OwnerDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Err(ENODEV) + } + + fn as_raw(&self) -> *mut bindings::drm_file { + unreachable!("KUnit object tests do not open DRM files") + } + } + + #[vtable] + impl gem::DriverObject for OwnerObject { + type Driver = OwnerDriver; + type Args = PageOwner; + + fn new( + _dev: &drm::Device, + _size: usize, + args: Self::Args, + ) -> impl PinInit { + let fail = args.fail; + // A failure unwinds this initializer, which drops the owner it already took. + try_pin_init!(OwnerObject { + _owner: args, + _: { + if fail { + return Err(ENOMEM); + } + }, + }) + } + } + + #[vtable] + impl drm::Driver for OwnerDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = OwnerFile; + type Object = Object; + type ParentDevice = faux::Device; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + // The pages go with the driver data. Whether the object is freed or its creation fails in the + // driver's initializer, no device mapping of them is left at that moment. + #[test] + fn external_mapping_never_outlives_the_pages() -> Result { + let reg = faux::Registration::new(c"OwnerKunit", None)?; + let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: As in `create_drm_dev()`. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } + let drm = UnregisteredDevice::::new(fdev, try_pin_init!(KunitData {}))?; + for fail in [true, false] { + let pages = alloc_pages::<2>()?; + let raw = page_pointers(&pages); + MAPPED_AT_RELEASE.store(usize::MAX, Ordering::Release); + // SAFETY: The driver data holds the pages, and `raw` outlives the call. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: pages, + fail, + }, + ) + }; + if fail { + assert_eq!(obj.err(), Some(ENOMEM)); + } else { + let obj = obj?; + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 1); + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), usize::MAX); + drop(obj); + } + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), 0); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + } + Ok(()) + } + + // A device that cannot address one of an object's pages. Left to itself the DMA layer gives + // such a device a bounce buffer in the page's place, which the test first shows with an + // ordinary mapping of that page. An object over the page must fail instead, with nothing + // left mapped when its pages are released. + // + // Built for arm64, which always sets up bounce buffers (it bounces small kmalloc() buffers + // through them), so that the ordinary mapping has one to get. + #[cfg(CONFIG_ARM64)] + #[test] + fn external_mapping_refuses_a_bounce_buffer() -> Result { + let reg = faux::Registration::new(c"BounceKunit", None)?; + let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + let reachable = alloc_pages::<2>()?; + // The page the device cannot address is the lower of two, and comes second in its + // object: the two never form one segment, and the first is mapped before the second + // fails. + let [a, b] = alloc_pages::<2>()?; + let (near, hidden) = if a.phys() > b.phys() { (a, b) } else { (b, a) }; + let start = hidden.phys(); + let end = start + PAGE_SIZE as u64; + // SAFETY: As in `create_drm_dev()`. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } + // The bus of this device reaches all memory, at its CPU address, except `hidden`. + // SAFETY: The faux device is private to this test and has no mapping yet. + unsafe { + set_bus_map( + dev, + [ + bindings::bus_dma_region { + cpu_start: end, + dma_start: end, + size: u64::MAX - end, + }, + bindings::bus_dma_region { + cpu_start: 0, + dma_start: 0, + size: start, + }, + BUS_MAP_END, + ], + )? + }; + let drm = UnregisteredDevice::::new(fdev, try_pin_init!(KunitData {}))?; + + // An ordinary mapping of `hidden` for this device succeeds, and is a bounce buffer: its + // DMA address, which for this bus is a CPU address, lies outside the page, the memory + // there holds what the page held when it was mapped, and a later store to the page + // does not show there. + let (before, after) = (0xa5u8, 0x5au8); + // SAFETY: The page is live and private to this test, as it is below. + unsafe { hidden.write_raw(&before, 0, 1)? }; + let sgt = KBox::new(Opaque::::zeroed(), GFP_KERNEL)?; + let one = [hidden.as_ptr()]; + // SAFETY: `sgt` is zeroed memory for a table and `one` holds one valid page. + to_result(unsafe { + bindings::sg_alloc_table_from_pages_segment( + sgt.get(), + one.as_ptr().cast_mut(), + 1, + 0, + PAGE_SIZE, + u32::MAX, + GFP_KERNEL.as_raw(), + ) + })?; + let dir = bindings::dma_data_direction_DMA_TO_DEVICE; + // SAFETY: The table was built above; `dev` is bound. + let mapped = + to_result(unsafe { bindings::dma_map_sgtable(dev.as_raw(), sgt.get(), dir, 0) }); + let mut bounce = None; + if mapped.is_ok() { + // SAFETY: The table is mapped until the unmap below. + let addr = unsafe { scatterlist::SGTable::from_raw(sgt.get()) } + .iter() + .next() + .map(|segment| segment.dma_address()); + // SAFETY: See above. + let stored = unsafe { hidden.write_raw(&after, 0, 1) }; + if let (Some(addr), Ok(())) = (addr, stored) { + let base = addr & !(PAGE_SIZE as u64 - 1); + let mut seen = 0u8; + // SAFETY: The memory the DMA layer mapped stays allocated until the unmap, and + // nothing else uses it. + let read = unsafe { + crate::page::Page::borrow_phys(&base) + .map(|page| page.read_raw(&mut seen, (addr - base) as usize, 1)) + }; + if let Some(Ok(())) = read { + bounce = Some((addr, seen)); + } + } + // SAFETY: The table was mapped above for `dev` with `dir`. + unsafe { bindings::dma_unmap_sgtable(dev.as_raw(), sgt.get(), dir, 0) }; + } + // SAFETY: The table was allocated above and is not mapped. + unsafe { bindings::sg_free_table(sgt.get()) }; + assert_eq!(mapped, Ok(())); + let (addr, seen) = bounce.ok_or(EINVAL)?; + assert!(addr < start || addr >= end); + assert_eq!(seen, before); + + // The device takes an object over pages it reaches. + let raw = page_pointers(&reachable); + // SAFETY: The driver data holds the pages, and `raw` outlives the call. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: reachable, + fail: false, + }, + ) + }?; + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 1); + drop(obj); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + + // An object over `hidden` fails: the DMA layer refuses the mapping instead of bouncing + // the page. The driver data, which had taken the pages, releases them with no mapping + // left, the first page's included. + let raw = [near.as_ptr(), hidden.as_ptr()]; + MAPPED_AT_RELEASE.store(usize::MAX, Ordering::Release); + // SAFETY: As above. + let obj = unsafe { + Object::::new_external( + &drm, + dev, + &raw[..], + true, + ObjectConfig::default(), + PageOwner { + _pages: [near, hidden], + fail: false, + }, + ) + }; + assert_eq!(obj.err(), Some(EIO)); + assert_eq!(MAPPED_AT_RELEASE.load(Ordering::Acquire), 0); + assert_eq!(EXTERNAL_MAPPINGS.load(Ordering::Acquire), 0); + Ok(()) + } + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 6fb32e0396fa9..5ec4864f5eb22 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -25,6 +25,7 @@ TARGETS += drivers/dma-buf TARGETS += drivers/gpu/apple-afkv2 TARGETS += drivers/gpu/apple-dcp TARGETS += drivers/gpu/asahi-m4 +TARGETS += drivers/gpu/asahi-userptr TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice TARGETS += drivers/net diff --git a/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile b/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile new file mode 100644 index 0000000000000..1bd98ee6745dc --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-userptr/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CFLAGS += -O2 -Wall $(KHDR_INCLUDES) +TEST_GEN_PROGS := userptr +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c b/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c new file mode 100644 index 0000000000000..abb5ba2f5402c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-userptr/userptr.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * DRM_IOCTL_ASAHI_GEM_USERPTR on the GPU's render node: what it accepts and + * refuses, and the accounting (VmPin in /proc/self/status, the client's + * drm-total-system in its fdinfo). No GPU work is submitted. Skipped where + * the render node is not this driver's or lacks the feature, and where the + * caller has neither CAP_IPC_LOCK nor a RLIMIT_MEMLOCK hard limit with room + * for the four pages a test pins. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../kselftest_harness.h" + +#define PAGE 0x4000UL + +static int ioc(int fd, unsigned long cmd, void *arg) +{ + int ret; + + do { + ret = ioctl(fd, cmd, arg); + } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + return ret ? -errno : 0; +} + +/* VmPin of this process (mm->pinned_vm) in kB. */ +static long vm_pin_kb(void) +{ + char line[256]; + long kb = -1; + FILE *f = fopen("/proc/self/status", "r"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %ld kB", &kb) == 1) + break; + fclose(f); + return kb; +} + +/* drm-total-system of the client behind @fd, in bytes: the objects it created. */ +static long long client_total(int fd) +{ + char path[64], line[256], unit[8]; + long long value = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", fd); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unit[0] = 0; + if (sscanf(line, "drm-total-system: %lld %7s", &value, unit) < 1) + continue; + if (!strcmp(unit, "KiB")) + value <<= 10; + else if (!strcmp(unit, "MiB")) + value <<= 20; + break; + } + fclose(f); + return value; +} + +static int userptr(int fd, void *ptr, uint64_t size, uint32_t flags, + uint32_t vm, uint32_t *handle) +{ + struct drm_asahi_gem_userptr args = { + .userptr = (uintptr_t)ptr, + .size = size, + .flags = flags, + .vm_id = (flags & DRM_ASAHI_GEM_VM_PRIVATE) ? vm : 0, + }; + int ret = ioc(fd, DRM_IOCTL_ASAHI_GEM_USERPTR, &args); + + if (!ret) + *handle = args.handle; + return ret; +} + +static int bind(int fd, uint32_t vm, uint32_t handle, uint64_t addr, + uint64_t size, uint32_t flags) +{ + struct drm_asahi_gem_bind_op op = { + .flags = flags, .handle = handle, .range = size, .addr = addr, + }; + struct drm_asahi_vm_bind args = { + .vm_id = vm, .num_binds = 1, .stride = sizeof(op), + .userptr = (uintptr_t)&op, + }; + + return ioc(fd, DRM_IOCTL_ASAHI_VM_BIND, &args); +} + +static int unbind(int fd, uint32_t vm, uint64_t addr, uint64_t size) +{ + return bind(fd, vm, 0, addr, size, DRM_ASAHI_BIND_UNBIND); +} + +static void close_handle(int fd, uint32_t handle) +{ + struct drm_gem_close args = { .handle = handle }; + + ioc(fd, DRM_IOCTL_GEM_CLOSE, &args); +} + +/* Whether the driver behind @fd is this one. */ +static int is_asahi(int fd) +{ + char name[16] = ""; + struct drm_version version = { + .name = name, .name_len = sizeof(name) - 1, + }; + + return !ioc(fd, DRM_IOCTL_VERSION, &version) && !strcmp(name, "aurora"); +} + +/* + * Clears CAP_IPC_LOCK in the effective set of this process, which is always + * allowed. Returns 0 once the process is without it. + */ +static int drop_cap_ipc_lock(void) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + }; + struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3]; + + if (syscall(SYS_capget, &header, data)) + return -errno; + data[CAP_TO_INDEX(CAP_IPC_LOCK)].effective &= ~CAP_TO_MASK(CAP_IPC_LOCK); + return syscall(SYS_capset, &header, data) ? -errno : 0; +} + +/* The largest range a test binds, and the most one pins outside memlock_limit. */ +#define BIND_MAX (4 * PAGE) + +/* + * Whether this process may pin BIND_MAX more: it has CAP_IPC_LOCK, or + * RLIMIT_MEMLOCK has room for it, after raising the soft limit to the hard + * one if that takes it. + */ +static int memlock_room(void) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + }; + struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3]; + long pin = vm_pin_kb(); + uint64_t need = (pin > 0 ? (uint64_t)pin << 10 : 0) + BIND_MAX; + struct rlimit lim; + + if (!syscall(SYS_capget, &header, data) && + (data[CAP_TO_INDEX(CAP_IPC_LOCK)].effective & + CAP_TO_MASK(CAP_IPC_LOCK))) + return 1; + if (getrlimit(RLIMIT_MEMLOCK, &lim)) + return 0; + if (lim.rlim_cur == RLIM_INFINITY || lim.rlim_cur >= need) + return 1; + if (lim.rlim_max != RLIM_INFINITY && lim.rlim_max < need) + return 0; + lim.rlim_cur = lim.rlim_max; + return !setrlimit(RLIMIT_MEMLOCK, &lim); +} + +/* + * A GPU address at which BIND_MAX bytes can be bound. The driver keeps + * ranges of every VM for its own mappings, where a bind fails with EINVAL, + * and GET_PARAMS does not list them: take the first address, in steps of + * 4 GiB below the kernel range, at which the ordinary buffer object @handle + * (of BIND_MAX bytes) binds. + */ +static uint64_t find_va(int fd, uint32_t vm, uint32_t handle, uint64_t start, + uint64_t end) +{ + uint64_t va; + + for (va = start + (4ULL << 30); va + BIND_MAX <= end; va += 4ULL << 30) { + if (bind(fd, vm, handle, va, BIND_MAX, DRM_ASAHI_BIND_READ)) + continue; + return unbind(fd, vm, va, BIND_MAX) ? 0 : va; + } + return 0; +} + +FIXTURE(asahi) +{ + int fd; + struct drm_asahi_params_global params; + uint32_t vm; + uint64_t va; +}; + +FIXTURE_SETUP(asahi) +{ + struct drm_asahi_get_params get = { + .pointer = (uintptr_t)&self->params, + .size = sizeof(self->params), + }; + + self->fd = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC); + if (self->fd < 0 || !is_asahi(self->fd)) + SKIP(return, "renderD128 is not an asahi render node"); + memset(&self->params, 0, sizeof(self->params)); + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GET_PARAMS, &get)); + if (!(self->params.features & DRM_ASAHI_FEATURE_USERPTR)) + SKIP(return, "no DRM_ASAHI_FEATURE_USERPTR"); + if (!memlock_room()) + SKIP(return, "RLIMIT_MEMLOCK has no room for the %lu bytes a test pins", + BIND_MAX); + + uint64_t kernel = self->params.vm_kernel_min_size > (32ULL << 30) ? + self->params.vm_kernel_min_size : 32ULL << 30; + struct drm_asahi_vm_create vm = { + .kernel_start = self->params.vm_end - kernel, + .kernel_end = self->params.vm_end, + }; + /* Lives as long as the file, so that the client's total stays put. */ + struct drm_asahi_gem_create probe = { + .size = BIND_MAX, .flags = DRM_ASAHI_GEM_VM_PRIVATE, + }; + + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_VM_CREATE, &vm)); + self->vm = vm.vm_id; + probe.vm_id = self->vm; + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_CREATE, &probe)); + self->va = find_va(self->fd, self->vm, probe.handle, + self->params.vm_start, vm.kernel_start); + ASSERT_NE(0, self->va); +} + +FIXTURE_TEARDOWN(asahi) +{ + close(self->fd); +} + +TEST_F(asahi, anonymous_memory_round_trip) +{ + const uint64_t size = 4 * PAGE; + long pin0 = vm_pin_kb(); + long long total0 = client_total(self->fd); + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint32_t h = 0; + + ASSERT_NE(MAP_FAILED, mem); + ASSERT_GE(total0, 0); + ASSERT_EQ(0, userptr(self->fd, mem, size, DRM_ASAHI_GEM_VM_PRIVATE, + self->vm, &h)); + EXPECT_EQ(pin0 + (long)(size >> 10), vm_pin_kb()); + EXPECT_EQ(total0 + (long long)size, client_total(self->fd)); + EXPECT_EQ(0, bind(self->fd, self->vm, h, self->va, size, + DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE)); + + /* No CPU mapping, advice, export or special binding through DRM. */ + struct drm_asahi_gem_mmap_offset off = { .handle = h }; + struct drm_asahi_gem_madvise madv = { + .handle = h, .madv = DRM_ASAHI_GEM_MADV_DONTNEED, + }; + struct drm_prime_handle prime = { .handle = h, .flags = DRM_CLOEXEC }; + struct drm_asahi_gem_bind_object ts = { + .op = DRM_ASAHI_BIND_OBJECT_OP_BIND, + .flags = DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS, + .handle = h, .range = PAGE, + }; + + EXPECT_EQ(-EINVAL, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, &off)); + EXPECT_EQ(-EINVAL, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MADVISE, &madv)); + EXPECT_NE(0, ioc(self->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime)); + EXPECT_NE(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_BIND_OBJECT, &ts)); + + /* Unmapped while bound: the object keeps the pages until it goes. */ + ASSERT_EQ(0, munmap(mem, size)); + EXPECT_EQ(pin0 + (long)(size >> 10), vm_pin_kb()); + EXPECT_EQ(0, unbind(self->fd, self->vm, self->va, size)); + close_handle(self->fd, h); + EXPECT_EQ(pin0, vm_pin_kb()); + EXPECT_EQ(total0, client_total(self->fd)); +} + +TEST_F(asahi, bad_ranges) +{ + char *mem = mmap(NULL, 3 * PAGE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + long pin0 = vm_pin_kb(); + uint32_t h = 0; + + ASSERT_NE(MAP_FAILED, mem); + ASSERT_EQ(0, munmap(mem + 2 * PAGE, PAGE)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem + 4096, PAGE, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE + 4096, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, 0, 0, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE, 1u << 0 /* WRITEBACK */, 0, &h)); + EXPECT_EQ(-EINVAL, userptr(self->fd, mem, PAGE, 1u << 7, 0, &h)); + /* Two pages pin, the third is a hole: nothing stays pinned. */ + EXPECT_EQ(-EFAULT, userptr(self->fd, mem, 3 * PAGE, 0, 0, &h)); + EXPECT_EQ(pin0, vm_pin_kb()); + /* A CPU mapping of another buffer object is a PFN mapping. */ + struct drm_asahi_gem_create bo = { .size = PAGE }; + struct drm_asahi_gem_mmap_offset off = { 0 }; + void *bo_map; + + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_CREATE, &bo)); + off.handle = bo.handle; + ASSERT_EQ(0, ioc(self->fd, DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET, &off)); + bo_map = mmap(NULL, PAGE, PROT_READ | PROT_WRITE, MAP_SHARED, + self->fd, off.offset); + ASSERT_NE(MAP_FAILED, bo_map); + EXPECT_EQ(-EFAULT, userptr(self->fd, bo_map, PAGE, 0, 0, &h)); + EXPECT_EQ(-EFAULT, userptr(self->fd, bo_map, PAGE, + DRM_ASAHI_GEM_USERPTR_READ_ONLY, 0, &h)); + munmap(bo_map, PAGE); + close_handle(self->fd, bo.handle); + EXPECT_EQ(pin0, vm_pin_kb()); + munmap(mem, 2 * PAGE); +} + +/* A read-only shared mapping of a file the caller cannot write. */ +TEST_F(asahi, read_only_file_pages) +{ + const char *path = "/usr/bin/sh"; + int file = open(path, O_RDONLY | O_CLOEXEC); + uint32_t h = 0; + + ASSERT_GE(file, 0); + if (lseek(file, 0, SEEK_END) < (off_t)(2 * PAGE)) + SKIP(return, "%s is smaller than two pages", path); + void *mem = mmap(NULL, 2 * PAGE, PROT_READ, MAP_SHARED, file, 0); + + ASSERT_NE(MAP_FAILED, mem); + EXPECT_EQ(-EFAULT, userptr(self->fd, mem, 2 * PAGE, 0, 0, &h)); + ASSERT_EQ(0, userptr(self->fd, mem, 2 * PAGE, + DRM_ASAHI_GEM_USERPTR_READ_ONLY, 0, &h)); + EXPECT_EQ(-EACCES, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE)); + EXPECT_EQ(-EACCES, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_WRITE)); + EXPECT_EQ(0, bind(self->fd, self->vm, h, self->va, 2 * PAGE, + DRM_ASAHI_BIND_READ)); + EXPECT_EQ(0, unbind(self->fd, self->vm, self->va, 2 * PAGE)); + close_handle(self->fd, h); + munmap(mem, 2 * PAGE); + close(file); +} + +/* + * RLIMIT_MEMLOCK bounds the pages of a caller without CAP_IPC_LOCK. The test + * gives the capability up first (the harness runs each test in a process of + * its own), so a request above the limit has to fail whoever runs it. It + * lowers the soft limit to what the process has pinned already plus 16 pages, + * or as many as the hard limit has room for, and is skipped where that is + * less than two. + */ +TEST_F(asahi, memlock_limit) +{ + struct rlimit old, lim; + long pin0 = vm_pin_kb(); + uint64_t pinned = pin0 > 0 ? (uint64_t)pin0 << 10 : 0; + uint64_t pages = 16, size; + uint32_t h = 0; + void *mem; + + ASSERT_EQ(0, drop_cap_ipc_lock()); + ASSERT_EQ(0, getrlimit(RLIMIT_MEMLOCK, &old)); + if (old.rlim_max != RLIM_INFINITY) { + uint64_t room = old.rlim_max > pinned ? + (old.rlim_max - pinned) / PAGE : 0; + + if (room < pages) + pages = room; + } + if (pages < 2) + SKIP(return, "no room for two pages in the RLIMIT_MEMLOCK hard limit of %llu bytes", + (unsigned long long)old.rlim_max); + size = 4 * pages * PAGE; + mem = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, mem); + lim.rlim_cur = pinned + pages * PAGE; + lim.rlim_max = old.rlim_max; + ASSERT_EQ(0, setrlimit(RLIMIT_MEMLOCK, &lim)); + ASSERT_EQ(-ENOMEM, userptr(self->fd, mem, size, 0, 0, &h)); + EXPECT_EQ(pin0, vm_pin_kb()); + EXPECT_EQ(0, userptr(self->fd, mem, pages / 2 * PAGE, 0, 0, &h)); + /* The pages already pinned count against the limit too. */ + EXPECT_EQ(-ENOMEM, userptr(self->fd, (char *)mem + pages / 2 * PAGE, + pages * PAGE, 0, 0, &h)); + close_handle(self->fd, h); + setrlimit(RLIMIT_MEMLOCK, &old); + munmap(mem, size); +} + +/* After fork the parent keeps its pinned pages; the child writes a copy. */ +TEST_F(asahi, fork_keeps_the_parent_pages) +{ + uint32_t *mem = mmap(NULL, PAGE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint32_t h = 0; + int status; + + ASSERT_NE(MAP_FAILED, mem); + mem[0] = 0x1234; + ASSERT_EQ(0, userptr(self->fd, mem, PAGE, 0, 0, &h)); + pid_t pid = fork(); + + ASSERT_GE(pid, 0); + if (pid == 0) { + mem[0] = 0xdead; + _exit(mem[0] == 0xdead ? 0 : 1); + } + ASSERT_EQ(pid, waitpid(pid, &status, 0)); + EXPECT_EQ(0, WEXITSTATUS(status)); + EXPECT_EQ(0x1234, mem[0]); + close_handle(self->fd, h); + munmap(mem, PAGE); +} + +TEST_HARNESS_MAIN