From 4b7aa809ff54c1fc60c91390ff9bfcab15943bb8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 19:13:02 -0400 Subject: [PATCH 01/68] drm/asahi: sync the G17 driver with aurora-silicon at d3cf19be Bring drivers/gpu/drm/asahi up to the aurora-silicon/linux state at d3cf19be, the base of the G17 performance series that follows, merged three-way on top of integration/aurora with the 7.2.10 merge point 44d5afc7 as base. integration/aurora's own work is kept: the G15 backend, the M3 T8122 runtime, UAT adoption, deferred-close binding sequences (#214) and the 7.2.11 shared-code adaptations. The G16 (T8132) backend is not carried. This carries the G17 work merged on the aurora side since 7.2.10, including the T6040 (M4 Pro) and 25G83 firmware support and the render replay/recovery series by Jonathan Goyvaerts, the watchdog, replay and backing-population changes by Ryan Murray, the tracked VM lifetime work by Eryk Wieliczko, and the G17 queue, capacity and timestamp changes by DjDeveloperr (aurora-silicon/linux #382, #391, #397, #400). Adaptations to 7.2.11: - module parameters return references; THIS_MODULE replaces kernel::module::this_module; a local raw_try_init replaces the newer pin-init helper; of_device_table! takes the MODULE_OF_TABLE name. - regs.rs keeps G17's sgx window as Devres, as 7.2 has no DevresIoMem. - The shared feature flags keep G17's opt-out of soft faults. - G15 and the M3 runtime use the shared timestamp::Buffer and the reserved-table aware root walk; omacom's G15 vm_closed hook becomes vm_destroyed, next to the G17 vm_closed. - Crate and helper pieces the driver needs: the shmem SG lease and pages_no_reclaim (drm_gem_shmem_helper), DRM file memory stats, fdinfo and is_current_master, and the mm RSS counter. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/asahi_core.rs | 14 + drivers/gpu/drm/asahi/driver.rs | 13 +- drivers/gpu/drm/asahi/file.rs | 153 ++- drivers/gpu/drm/asahi/g15/command.rs | 3 +- drivers/gpu/drm/asahi/g15/mod.rs | 6 +- drivers/gpu/drm/asahi/g15/uat.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 64 +- drivers/gpu/drm/asahi/g17/buffer/target.rs | 61 +- drivers/gpu/drm/asahi/g17/capacity.rs | 88 ++ drivers/gpu/drm/asahi/g17/command.rs | 50 +- drivers/gpu/drm/asahi/g17/completion.rs | 130 ++- drivers/gpu/drm/asahi/g17/context.rs | 45 +- drivers/gpu/drm/asahi/g17/context/identity.rs | 9 + drivers/gpu/drm/asahi/g17/data_segment.rs | 56 ++ drivers/gpu/drm/asahi/g17/dependency.rs | 8 + drivers/gpu/drm/asahi/g17/event.rs | 8 +- drivers/gpu/drm/asahi/g17/feed.rs | 23 +- drivers/gpu/drm/asahi/g17/freelist.rs | 141 ++- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 77 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 157 ++- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 681 +++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/queue.rs | 196 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 303 +++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 133 +++ drivers/gpu/drm/asahi/g17/initdata.rs | 202 ++-- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 238 +++++ drivers/gpu/drm/asahi/g17/job.rs | 71 +- drivers/gpu/drm/asahi/g17/kick.rs | 381 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 469 +++++++-- drivers/gpu/drm/asahi/g17/object.rs | 112 ++- drivers/gpu/drm/asahi/g17/power.rs | 123 +++ drivers/gpu/drm/asahi/g17/preparation.rs | 153 +++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 474 ++++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 456 +++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 3 + .../drm/asahi/g17/queue/compute/retirement.rs | 180 +++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 36 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 518 +++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 111 ++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 43 +- .../drm/asahi/g17/queue/render/publication.rs | 34 +- .../drm/asahi/g17/queue/render/retirement.rs | 8 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/recovery.rs | 35 +- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 84 ++ drivers/gpu/drm/asahi/g17/recovery/slot.rs | 8 +- drivers/gpu/drm/asahi/g17/replay.rs | 109 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 605 +++++++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 102 +- .../drm/asahi/g17/runtime/compute_control.rs | 18 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 19 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 306 +++++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 7 +- .../drm/asahi/g17/runtime/render/observe.rs | 6 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 8 + .../drm/asahi/g17/runtime/render/prepare.rs | 77 +- .../drm/asahi/g17/runtime/render/replay.rs | 148 +-- .../drm/asahi/g17/runtime/render/settle.rs | 16 +- .../drm/asahi/g17/runtime/render/submit.rs | 218 +++- .../drm/asahi/g17/runtime/render/teardown.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- drivers/gpu/drm/asahi/g17/teardown.rs | 15 + drivers/gpu/drm/asahi/g17/timeout.rs | 257 ++++- drivers/gpu/drm/asahi/g17/validation.rs | 5 +- drivers/gpu/drm/asahi/gem.rs | 68 +- drivers/gpu/drm/asahi/gpu.rs | 59 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t6040.rs | 180 ++++ drivers/gpu/drm/asahi/hw/t8140.rs | 37 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 5 +- drivers/gpu/drm/asahi/mem.rs | 42 +- drivers/gpu/drm/asahi/memacct.rs | 549 ++++++++++ drivers/gpu/drm/asahi/mmu.rs | 943 ++++++++++++++---- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 79 +- drivers/gpu/drm/asahi/mmu/m3.rs | 2 +- drivers/gpu/drm/asahi/mmu/ppl.rs | 71 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 69 +- drivers/gpu/drm/asahi/mmu/shared.rs | 4 +- drivers/gpu/drm/asahi/object.rs | 2 +- drivers/gpu/drm/asahi/pgtable.rs | 409 +++++--- drivers/gpu/drm/asahi/queue/common.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 20 +- drivers/gpu/drm/asahi/regs.rs | 64 +- drivers/gpu/drm/asahi/sysfs.c | 106 +- drivers/gpu/drm/asahi/sysfs.h | 3 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 21 +- drivers/gpu/drm/asahi/timestamp.rs | 185 ++++ drivers/gpu/drm/asahi/util.rs | 14 + drivers/gpu/drm/drm_gem_shmem_helper.c | 20 +- include/drm/drm_gem_shmem_helper.h | 11 +- include/uapi/drm/asahi_drm.h | 34 +- rust/bindings/bindings_helper.h | 1 + rust/helpers/mm.c | 12 + rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/file.rs | 69 ++ rust/kernel/drm/gem/shmem.rs | 487 ++++++--- rust/kernel/mm.rs | 53 + 104 files changed, 10248 insertions(+), 1518 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs create mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs create mode 100644 drivers/gpu/drm/asahi/g17/replay.rs create mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs create mode 100644 drivers/gpu/drm/asahi/memacct.rs create mode 100644 drivers/gpu/drm/asahi/mmu/ppl.rs create mode 100644 drivers/gpu/drm/asahi/timestamp.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0fbe6e728773d3..8b782a6f8ffa74 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -25,6 +25,7 @@ mod gpu; mod hw; mod initdata; mod mem; +mod memacct; mod m3; mod microseq; mod mmu; @@ -35,6 +36,7 @@ mod regs; mod slotalloc; mod stats; mod sysfs_exports; +mod timestamp; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -172,5 +174,17 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, + mem_limit_pct: u32 { + default: 67, + description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", + }, + client_mem_limit_pct: u32 { + default: 50, + description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", + }, + oom_attribution: u8 { + default: 1, + description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d8..9b9b8fc4ccd152 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -103,7 +103,6 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -153,6 +152,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -233,6 +236,7 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); + crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -419,6 +423,11 @@ impl AsahiPlatformDriver { return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + } Ok(AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, @@ -433,7 +442,7 @@ impl AsahiPlatformDriver { impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a5..c627bd9c6b8a05 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + memacct, + gpu, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -218,7 +223,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -236,6 +241,8 @@ pub(crate) struct File { /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, + /// GPU memory this client created and holds. + mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -255,9 +262,20 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } @@ -270,26 +288,37 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + crate::util::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + crate::util::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + crate::util::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); + addr_of_mut!((*slot).mem).write(mem); Ok(()) }) } @@ -341,7 +370,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +389,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -383,7 +415,7 @@ impl File { // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { @@ -402,15 +434,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +456,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +527,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +618,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -589,7 +641,7 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); + reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -632,8 +684,24 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = gem::user_pages_no_reclaim( + size, + reg_data.gpu.as_any().is::(), + ); + let gem = gem::new_object( + device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + )?; if reg_data.gpu.prepare_new_objects() { gem::prepare_new_object(&gem)?; } @@ -713,11 +781,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +802,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +820,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1268,27 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + return Err(EINVAL); + } + // SAFETY: capable() only reads the calling task's credentials (ioctl process context). + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + && !file.is_current_master() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1260,6 +1350,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1385,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1369,7 +1462,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026f..b343aa3fe4e5b5 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -360,7 +360,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 95f04114ba8748..48e6afa948c981 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -632,7 +632,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_destroyed(&self) { vm::closed(self) } @@ -658,7 +658,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 7ab3a1919eaecc..1d63fb83b9fa12 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -99,7 +99,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea02..8f3a03fbb8dba3 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } @@ -1119,3 +1131,51 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } + +impl BufferIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", + self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, + self.entries.iter().filter(|entry| entry.owner).count(), + self.entries.iter().filter(|entry| entry.references != 0).count()) + } +} +impl MetricsIds { + pub(crate) fn used(&self) -> u32 { + self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() + - (128 - fw::METRICS_SLOTS as u32) + } +} + +#[kunit_tests(asahi_g17_buffer_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_build_and_fault_retirement_return_parameter_buffers() { + let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); + for _ in 0..BUFFER_IDS { + clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); + } + assert_eq!(ids.allocate_render(), Err(ENOSPC)); + for (index, id) in clients.into_iter().enumerate() { + if index % 2 == 0 { + // Failed constructor never installed the graph owner. + ids.release(id).unwrap(); + } else { + ids.reserve_owner(id).unwrap(); + ids.release(id).unwrap(); + ids.acquire_owner(id).unwrap(); + // Close cannot return a still-owned failed command. + ids.unreserve_owner(id).unwrap(); + assert!(!ids.entries[id as usize].free()); + ids.release(id).unwrap(); + } + } + assert!(ids.entries.iter().all(|entry| entry.free())); + assert_eq!(ids.returned_count, BUFFER_IDS); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c935..64c88da8eca149 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at +/// the default parameter-buffer sizes. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs new file mode 100644 index 00000000000000..bcbbb2e7e875db --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Name capacity refusals at their source, before a common submit or fence +//! failure handler loses their provenance. Counters are device-lifetime +//! independent diagnostics, never admission or ownership state. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Site { + ComputeReserve, + PhysicalQids, + RenderSlots, + LogicalQueues, + ComputeSlots, + QosIds, + ComputeItems, + RetiredRenderBindings, + RenderPoolIds, + RenderWorkSlots, + ParameterBuffers, + MetricsIds, + LowerVa, + UpperVa, + AliasVa, + ExecutionContexts, +} + +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; +static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; + +/// First and power-of-two refusals per site keep a pressure storm bounded in +/// the log. This does not change whether the caller retries or rejects work. +#[track_caller] +pub(super) fn refused(site: Site) -> Error { + let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); + if count.is_power_of_two() { + let at = core::panic::Location::caller(); + pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); + } + ENOSPC +} + +/// Read-only diagnostics hold the live device session while sampling its pools. +/// They never wake firmware, reclaim a resource, or change admission state. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), + u8::from(self.0.crashed.load(Ordering::Acquire)))?; + let uat = { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1")?; + firmware.queues.write_capacity(f)?; + firmware.pool.write_capacity(f)?; + writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; + firmware.uat.clone() + }; + // Mapping destruction can issue firmware flushes while holding VM + // locks. Never take a VM/TTBAT lock with the device mutex held here. + // The retained UAT owns valid allocator metadata even during shutdown. + writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let mut upper = uat.geometry().kernel_range(); + upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + for (name, vm, range) in [ + ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), + ("dynamic_upper_bytes", uat.kernel_vm(), upper), + ] { + match vm.capacity(range.clone()) { + Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, + Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, + } + } + for (site, count) in [ + Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, + Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, + Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, + ].iter().zip(REFUSALS.iter()) { + writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 3dfa760969224e..c65f7de6ac9157 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,6 +12,8 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; +const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); + const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -121,7 +123,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -131,7 +133,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if header.cmd_type as u32 + let payload = if u32::from(header.cmd_type) == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -238,7 +240,7 @@ impl UscWindow { } require_range( space, - self.base + (tagged_offset & !7) as u64, + self.base + u64::from(tagged_offset & !7), 4, Access::Read, ) @@ -306,12 +308,14 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { + long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -323,6 +327,19 @@ pub(crate) enum Validated { }, } +impl Validated { + pub(crate) fn long_job(&self) -> bool { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, + } + } + pub(crate) fn allow_long_job(&mut self) { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, + } + } +} + impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -334,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -407,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -440,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -447,6 +470,8 @@ impl Payload { } } Ok(Validated::Render { + long_job: super::timeout::select(false, + ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -454,8 +479,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 + let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; + if cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -470,14 +495,19 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { + long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -489,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231dba..4f5b87bc853dc0 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -28,7 +28,7 @@ pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, + _mapping: Arc, cpu: mmu::TimestampMapping, } @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,84 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + +/// Republications of one command whose earlier publications firmware dropped unexecuted. +pub(crate) const MAX_REPLAYS: u8 = 3; + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +252,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -185,11 +264,18 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, + /// A recovery blamed this command. Unlike `class`, which a later + /// classification can change, this never clears. + blamed: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { + #[expect( + clippy::too_many_arguments, + reason = "Completion construction takes distinct queue, context, and lifetime owners." + )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -216,7 +302,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -235,6 +321,7 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), + blamed: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -278,14 +365,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -305,7 +392,7 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < 3).then_some(count + 1) + (count < MAX_REPLAYS).then_some(count + 1) }) .is_ok(); if accepted { @@ -335,7 +422,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -384,6 +471,20 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } + pub(crate) fn guilty(&self) -> bool { + self.class.load(Ordering::Acquire) == 2 + } + + /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, + /// so it is never published again. + pub(crate) fn note_blamed(&self) { + self.blamed.store(true, Ordering::Release); + } + + pub(crate) fn blamed(&self) -> bool { + self.blamed.load(Ordering::Acquire) + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -401,6 +502,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -411,7 +518,7 @@ impl Completion { /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(); + lease.quarantine(self.spared()); } } @@ -442,8 +549,11 @@ impl Completion { feed.note_render_pass(ticks[2], ticks[3]); } if let Err(error) = result { + // Spared work cancelled after another VM's recovery is not this client's device loss. if self.spared() { - self.status.report_failure(error); + if error != ECANCELED { + self.status.report_failure(error); + } } else if error != ECANCELED { self.status.record(error); } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44beef..c93ab25d55ce34 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; Ok(QosLease { @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -194,12 +196,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -387,7 +393,8 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context.work.lock().render.get(pair).ok_or_else(|| + super::capacity::refused(super::capacity::Site::RenderWorkSlots)) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -406,3 +413,37 @@ impl Drop for WorkStateLease { } } } + +impl QosIds { + pub(crate) fn used(&self) -> u32 { + (*self.used.lock() & !3u128).count_ones() + } +} + +#[kunit_tests(asahi_g17_qos_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_client_construction_returns_every_qos_id() { + let ids = QosIds::new().unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); + for _ in 0..126 { + clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 126); + assert!(matches!(ids.acquire(), Err(ENOSPC))); + drop(clients); + assert_eq!(ids.used(), 0); + assert_eq!(*ids.used.lock(), 3); + } + } +} + +impl Context { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { + writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", + scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 6c48367e2d6864..595c7154d42edc 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -69,3 +69,12 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } + +impl WorkStates { + pub(super) fn used(&self) -> usize { + self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() + } +} +impl RenderSlots { + pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } +} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs new file mode 100644 index 00000000000000..02bcc66de8f51c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/data_segment.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2026 The Gravity Linux Contributors +// +// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and +// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. + +//! Crash log buffers in the GPU firmware's data segment. +//! +//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it +//! by physical address inside its data segment, the `fw-data` reserved region, which the boot +//! loader keeps for the loaded firmware until reboot. + +use kernel::{ + c_str, device, + io::{ + self, + IoSysMap, + Region, // + }, + prelude::*, +}; + +/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. +pub(super) struct DataSegmentLog { + mem: io::mem::Mem, + offset: usize, + size: usize, +} + +impl DataSegmentLog { + /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which + /// must lie inside the `fw-data` region of `dev`. + pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { + return Err(EINVAL); + } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + Ok(Self { mem, offset, size }) + } + + /// The CPU view of the buffer. + pub(super) fn buf(&mut self) -> Result> { + // SAFETY: map() checked that the range lies inside the mapping. + let base = unsafe { self.mem.ptr().add(self.offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5abd494eeb1f58..0bc1d5024566e7 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -17,6 +17,14 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// A replay's original host inputs must still be signalled. Treat an unmet +/// input as a failed replay, never as a wait under the device mutex. +pub(crate) fn replay_status(status: i32) -> Result { + if status == 0 { Err(EBUSY) } + else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else { Ok(()) } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c813..9508a827be4b1b 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 4cd05fde165812..0045dff413922f 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -147,6 +147,10 @@ impl Requests { } } + /// Add a policy to the pinned requests. + /// + /// # Safety + /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -179,7 +183,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } } @@ -193,7 +197,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + bindings::FREQ_QOS_MIN_DEFAULT_VALUE }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -211,7 +215,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } applied } @@ -235,7 +239,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } fn clear(mut self: Pin<&mut Self>) { @@ -364,6 +368,13 @@ impl Feed { } } + /// Process a cpufreq policy notification. + /// + /// # Safety + /// + /// `block` must be this pinned Feed's registered notifier, and `data` must + /// point to the live policy supplied by the cpufreq notifier core. + /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -387,12 +398,12 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as _; + return bindings::NOTIFY_DONE as i32; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as _ + bindings::NOTIFY_OK as i32 } fn now() -> u64 { diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc0..3b31c3cead064a 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) } } impl Drop for RenderId { @@ -69,19 +69,84 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -100,7 +165,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - fn page_list_va(&self) -> u64 { + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -144,14 +209,17 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, @@ -522,6 +590,18 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } + self.write_pages(&pages)?; + self.pages = Some(pages); + Ok(()) + } + /// Preserve both sets of pages on failure. The caller owns the consumed + /// release and returns old pages for destruction outside the device mutex. + pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { + self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; + core::mem::swap(&mut self.pages, pages); + Ok(()) + } + fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -540,9 +620,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); - self.pages = Some(pages); Ok(()) } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -742,3 +822,30 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } + +impl RenderIds { + pub(crate) fn used(&self) -> u32 { + self.used[0].load(Ordering::Acquire).count_ones() + + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() + } +} + +#[kunit_tests(asahi_g17_render_id_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn rejected_pool_construction_returns_all_render_ids() { + let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); + for _ in 0..127 { + pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 127); + assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); + drop(pools); + assert_eq!(ids.used(), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index ec6797637bdbe1..9d430d50db365e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c4cf6960db296d..524132c4ae7beb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -498,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: first as u8, + first: u8::from(first), }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0633da7f435411..0cb7304e306776 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u16::MAX as u64; - let cliques = (layout >> 24) & u8::MAX as u64; - let user = (layout >> 32) & u16::MAX as u64; - let driver = (layout >> 48) & u16::MAX as u64; + let depth = layout & u64::from(u16::MAX); + let cliques = (layout >> 24) & u64::from(u8::MAX); + let user = (layout >> 32) & u64::from(u16::MAX); + let driver = (layout >> 48) & u64::from(u16::MAX); if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,13 +152,42 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster of six cores (A18 Pro). + pub(crate) const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + pub(crate) const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +312,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +336,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -347,14 +379,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +396,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 1f25bd0fc1c0dc..3092e599caa846 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -591,31 +608,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va - .checked_add(self.size as u64)? + let start = va & !mask; + let span = va + .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +657,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +678,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -676,14 +704,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +761,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +791,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) @@ -1169,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1197,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1213,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1224,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1234,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1285,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1319,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1364,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1411,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 00000000000000..c237896a49fb1a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index e07fbfb485a10a..4f3a70671eb87d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -297,7 +297,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, }) } @@ -332,7 +332,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | ((self.count - 1) as u64) << Self::COUNT_SHIFT + | u64::from(self.count - 1) << Self::COUNT_SHIFT } } @@ -444,8 +444,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | (args.qid as u64) << Self::QID_SHIFT - | (args.priority as u64) << Self::PRIORITY_SHIFT; + | u64::from(args.qid) << Self::QID_SHIFT + | u64::from(args.priority) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -478,9 +478,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: mask as u64 - | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT - | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + dependency_control: u64::from(mask) + | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT + | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index b6bb6aaea860ff..1cb46fce2d6028 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -30,7 +33,7 @@ pub(crate) mod qos; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; + unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 48b110a4193e3b..5708a2251a062d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -94,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -115,26 +121,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +152,62 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + /// Offsets of the two host words a [`PointerLayout`] may name. + const WORD_40: usize = core::mem::offset_of!(Self, word_40); + const WORD_50: usize = core::mem::offset_of!(Self, word_50); + + /// Whether `offset` is one of the host words a [`PointerLayout`] may name. + const fn is_host_word(offset: usize) -> bool { + offset == Self::WORD_40 || offset == Self::WORD_50 + } + + /// Whether `layout` names host words of the block, and two different ones when it has a + /// recovery target. Every firmware variant's layout is checked at build time. + pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { + match layout.recovery_target { + None => Self::is_host_word(layout.producer), + Some(target) => { + Self::is_host_word(layout.producer) + && Self::is_host_word(target) + && target != layout.producer + } + } + } /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Result { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target)?.store(u32::MAX, Ordering::Relaxed); + } + Ok(block) + } + + /// The host word at `offset`, which must be one a [`PointerLayout`] may name. + fn word(&self, offset: usize) -> Result<&AtomicU32> { + match offset { + Self::WORD_40 => Ok(&self.word_40), + Self::WORD_50 => Ok(&self.word_50), + _ => Err(EINVAL), } } @@ -174,12 +216,14 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { + let producer = self.word(layout.producer)?; self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + producer.store(base, Ordering::Relaxed); + Ok(()) } } @@ -271,12 +315,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [args.policy.priority() as u32; 2], + priority: [u32::from(args.policy.priority()); 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: args.policy.policy() as u32, + policy: u32::from(args.policy.policy()), unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -307,12 +351,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [policy.priority() as u32; 2]; + let priority = [u32::from(policy.priority()); 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != policy.policy() as u32 { - self.policy = policy.policy() as u32; + if { self.policy } != u32::from(policy.policy()) { + self.policy = u32::from(policy.policy()); } } } @@ -480,7 +524,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - binding.fragment_qid as u32, + u32::from(binding.fragment_qid), ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -491,14 +535,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: args.qid as u16, - install: args.install as u16, + qid: u16::from(args.qid), + install: u16::from(args.install), unk_1c: 0, - priority: args.policy.priority() as u16, + priority: u16::from(args.policy.priority()), unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed_valid: u32::from(args.completion_seed.is_some()), completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -506,13 +550,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: args.qos_update as u32, - qos_slot: args.qos_slot as u16, + qos_update: u32::from(args.qos_update), + qos_slot: u16::from(args.qos_slot), unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: args.context_update as u16, + context_update: u16::from(args.context_update), unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -525,6 +569,94 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp, in addition to the kick-entry dependency. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u64, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id, + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -564,7 +696,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: qid as u16, + qid: u16::from(qid), count: 1, priority, timestamp: timestamp.get(), @@ -603,7 +735,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: qid as u16, + qid: u16::from(qid), previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 85a2385b421b81..0e270d40392a88 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,53 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// One cluster (A18 Pro). + pub(crate) const ONE_CLUSTER: Deflake = Deflake { + sub: [0x20, 0x2a0], + viewport: true, + }; + /// Two clusters (M4 Pro). + pub(crate) const TWO_CLUSTERS: Deflake = Deflake { + sub: [0xa0, 0x820], + viewport: false, + }; +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +147,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +309,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +364,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -357,12 +411,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = pass.width as u64; - let height = pass.height as u64; + let width = u64::from(pass.width); + let height = u64::from(pass.height); let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / pass.utile_width as u64; - let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles_x = TILE_SIZE / u64::from(pass.utile_width); + let utiles_y = TILE_SIZE / u64::from(pass.utile_height); let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -411,7 +465,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -441,19 +495,19 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u32::MAX as u64, + false => spec & u64::from(u32::MAX), }; - let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); Ok(Self { - utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | pass.samples_log2 as u64, + utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | u64::from(pass.samples_log2), tile_config, layer_mode, - tib_blocks: (pass.sample_size as u64 - * pass.utile_width as u64 - * pass.utile_height as u64 + tib_blocks: (u64::from(pass.sample_size) + * u64::from(pass.utile_width) + * u64::from(pass.utile_height) * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, @@ -491,9 +545,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let deflake = user_offset32(r.deflake_va)? as u64; - let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; - let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let [deflake_20, deflake_2a0] = args.deflake.sub; + let deflake = u64::from(user_offset32(r.deflake_va)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -561,7 +616,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -573,7 +628,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -581,6 +636,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +766,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -609,8 +792,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, p.merge_upper_x as u64), - RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), + RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -625,15 +808,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, p.depth_clear as u64), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15301, u64::from(p.depth_clear)), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -682,11 +865,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -716,7 +899,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -742,13 +925,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -760,13 +943,13 @@ fn partial_load_registers( RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -902,7 +1085,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,7 +1103,11 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { let mut trimmed = [registers[0]; TA_REGISTERS]; @@ -945,7 +1133,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -969,7 +1160,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = fragment_busy as u32; + self.fragment_busy = u32::from(fragment_busy); self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -980,11 +1171,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1365,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,7 +1448,12 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; @@ -1289,12 +1493,15 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = (args.ordinal == 0) as u32; + desc.first_kick = u32::from(args.ordinal == 0); desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 00000000000000..2f7b5987333688 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, +} + +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, + init_ack: InitAck::Message, + power_on_done: true, + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, + lead_commands: false, + programs: super::render::RegisterPrograms::G17P, +}; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; + +const _: () = { + assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); +}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09eb..10af261aeb3acf 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -121,7 +123,7 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - self.record as *const _ as *const u8, + core::ptr::from_ref(self.record).cast::(), self.destination, core::mem::size_of_val(self.record), ); @@ -155,6 +157,15 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -174,6 +185,8 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, + /// GPU clusters, as the GPU reports them. + clusters: u32, } impl BuildEnv<'_> { @@ -235,6 +248,11 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +265,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -363,7 +385,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); } Ok(()) })?; @@ -502,7 +524,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -522,7 +544,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -601,7 +623,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -694,7 +716,12 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + clusters: u32, + active_cores: u32, + variant: &super::fw::Variant, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -703,9 +730,13 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + clusters, }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; + if variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } + self.map_register_windows(&env, cfg.windows)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -713,31 +744,41 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -939,12 +980,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for slot in 0..2 { + for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - completion_rings[slot] = abi::CompletionRing { + *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -988,13 +1029,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1002,7 +1043,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1050,8 +1091,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1107,7 +1148,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1150,7 +1191,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1180,37 +1221,65 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + /// The GPU's operating points: `count` of them, each with one voltage cell for the whole + /// GPU or one per cluster. + fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + let cells = opps.first().map_or(0, |opp| opp.cells); + if cells != 1 && u32::try_from(cells) != Ok(clusters) { + dev_err!( + dev.as_ref(), + "GPU operating points have {} voltage cells, expected 1 or {}\n", + cells, + clusters + ); + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1218,7 +1287,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1247,9 +1316,14 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1261,7 +1335,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1336,7 +1410,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1351,8 +1425,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1363,20 +1441,20 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; @@ -1391,21 +1469,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1417,17 +1495,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1437,7 +1519,7 @@ impl super::recovery::Memory for InitData { fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); let mut entries = super::recovery::Entries::default(); for index in 0..256 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 00000000000000..85aa6468fa311f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6dd..61b8c3b7b47373 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -143,23 +143,40 @@ impl Packet { } } + /// Replay holds the device mutex. Its original inputs were satisfied before + /// first publication; never sleep on a fence that needs event service here. + pub(crate) fn check_replay_dependencies(&self) -> Result { + for fence in &self.inputs { + super::dependency::replay_status(fence_status(fence))?; + } + for fence in &self.firmware { check_status(fence)?; } + Ok(()) + } + /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { + let polls = if self.command.long_job() { 5 } else { 1 }; + let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); + loop { + if self.cancel_requested() || self.completion.status().get() != 0 { + return Err(ECANCELED); + } + if fence_status(fence) != 0 { break; } + if super::now_ns() >= deadline { return Err(ECANCELED); } + // SAFETY: The retained fence owns the pointer throughout the bounded wait. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::time::msecs_to_jiffies(10) as isize, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited > 0 { break; } } check_status(fence)?; } @@ -179,6 +196,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -201,7 +225,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |_| 1), + result.map_or_else(|error| error.to_errno(), |()| 1), Ordering::Release, ); } @@ -242,7 +266,8 @@ pub(crate) trait Backend: Send + Sync { None } - /// Lazily create or rebind this logical queue's physical compute graph. + /// Validate compute sizing without reserving a physical channel. Queued + /// commands may wait on a host signal submitted after this ioctl returns. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -271,6 +296,7 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, + execution_renewals: u8, packet: Arc, } @@ -287,12 +313,14 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, + execution_renewals: 0, packet: self.packet.clone(), } } @@ -301,9 +329,14 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } + if super::timeout::renew_execution( + job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { - job.victim_renewals += 1; + if deferred { + job.victim_renewals = job.victim_renewals.saturating_add(1); } deferred } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86de..c337a8b298f193 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -17,8 +24,24 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Minimum render-channel capacity protected from compute-only allocation. +// Protect one tiling/fragment pair for each logical execution context, plus +// the two unpublished bootstrap reservations. The remaining ten compute +// channels are shared after exact retirement, rather than pinned per client. +const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +/// Logical queues that can each hold a render pair: what the render reserve leaves beside the +/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone +/// (every execution context with no floor). +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + +const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); + assert!(COMPUTE_CHANNELS >= 4); + assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); +}; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -47,19 +70,42 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - compute_publication: u64, + initial_render: [Option; 2], + publication: u64, leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - compute_publication: 0, + initial_render: [None; 2], + publication: 0, leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -83,16 +129,56 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + // The QIDs below the floor are never the host's; the reserve shrinks + // with them, so the compute channel count stays the same. + let total = QID_COUNT - self.floor; + let reserve = COMPUTE_RENDER_RESERVE - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| { + matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) + }) + .count(); + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let replacement = usize::from(initial.is_some()); + if total - self.leased + replacement + <= reserve.saturating_sub(render - replacement) + { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); + } } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) + .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { qid: index as u8, owner, @@ -102,14 +188,19 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. - pub(crate) fn next_compute_publication(&mut self) -> Result { - self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.compute_publication) + /// Ordered under the device mutex; no atomic operation on the submission path. Compute and + /// render publications share it, so a watchdog can order work across both engines. + pub(crate) fn next_publication(&mut self) -> Result { + self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.publication) } /// Call after preflighting the complete installation, before any address @@ -127,6 +218,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -140,6 +238,82 @@ impl QueueIds { } } +#[kunit_tests(asahi_g17_qid_accounting)] +mod accounting_tests { + use super::*; + + fn table() -> KBox { + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() + } + + #[test] + fn render_identities_satisfy_the_compute_reserve() { + let mut ids = table(); + for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + assert_eq!(ids.leased, QID_COUNT); + } + + #[test] + fn compute_cannot_consume_the_render_minimum() { + let mut ids = table(); + for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + ids.cancel_unpublished(tiling).unwrap(); + assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + } + + #[test] + fn initial_reservations_yield_but_published_ids_do_not() { + let mut ids = table(); + ids.reserve_initial_render().unwrap(); + for owner in 3..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); + let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); + assert_ne!(first.qid(), second.qid()); + assert!(ids.initial_render.iter().all(Option::is_none)); + assert_eq!(ids.leased, QID_COUNT); + ids.publish(first).unwrap(); + assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT); + ids.cancel_unpublished(second).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT - 1); + } + + #[test] + fn failed_second_render_id_returns_only_the_unpublished_first() { + let mut ids = table(); + for owner in 1..QID_COUNT as u64 { + let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + ids.publish(id).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); + ids.cancel_unpublished(first).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); + assert_eq!(replacement.qid(), first.qid()); + assert_ne!(replacement, first); + assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -147,10 +321,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -186,6 +444,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -220,14 +481,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -255,14 +547,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -272,19 +576,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -297,7 +601,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -353,3 +657,20 @@ impl Queue { }) } } + +impl QueueIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let mut engines = [0usize; 3]; + let mut published = 0; + for entry in self.entries.iter().flatten() { + engines[match entry.id.engine { + DataMaster::Tiling => 0, + DataMaster::Fragment => 1, + DataMaster::Compute => 2, + }] += 1; + published += usize::from(entry.published); + } + writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", + self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9da90d2b0843ab..2999daec47015d 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -23,6 +23,7 @@ mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; +mod data_segment; mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; @@ -35,13 +36,15 @@ mod ids; pub(crate) mod hazard; mod initdata; mod job; +mod capacity; mod kick; mod object; -mod power; +pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; +mod replay; mod runtime; pub(crate) mod status; mod teardown; @@ -149,12 +152,63 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, + /// Layouts of the command buffers that follow the GPU's core layout. + pub(crate) core_layout: CoreLayout, +} + +/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. +pub(crate) struct CoreLayout { + /// The cluster and enabled core counts the layouts were measured on, which the GPU must + /// report at probe; `None` where the layouts hold for every configuration of the SoC. + pub(crate) cores: Option<(u32, u32)>, + /// A compute queue's operand-state buffer. + pub(crate) operand_state: fw::compute::OperandState, + /// A render's deflake buffer. + pub(crate) deflake: fw::render::Deflake, +} + +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -197,6 +251,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -207,7 +264,15 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData(data_segment::DataSegmentLog), } impl rtkit::Buffer for CrashLog { @@ -216,7 +281,10 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData(log) => log.buf(), + } } } @@ -225,8 +293,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -305,13 +373,23 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { - return Err(EINVAL); + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); } + // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( + &data.shared.dev, + iova, + size, + )?), }) } } @@ -324,7 +402,7 @@ struct Coprocessor { running: bool, rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -338,27 +416,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), + }; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -404,8 +487,9 @@ impl Coprocessor { Pin::new(&mut self.rtkit).boot() } - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { let mut rtkit = Pin::new(&mut self.rtkit); rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; rtkit.as_mut().start_endpoint(EP_DOORBELL)?; @@ -413,7 +497,10 @@ impl Coprocessor { EP_FIRMWARE, MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), )?; - self.state.wait_for_ack() + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. @@ -452,6 +539,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -472,15 +567,21 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + hold: power::Hold, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -490,7 +591,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -500,6 +603,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -519,6 +631,11 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { + // An idle hold keeps the cores powered past registration; it releases them itself. + if self.hold.active() { + self.registration_power_release_pending = false; + return Ok(()); + } power::release(self) } @@ -548,14 +665,49 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - match self.effort.idle(now_ns, in_flight) { - power::IdleAction::None => Ok(None), + let effort = match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => None, power::IdleAction::Clear => { self.apply_effort(0)?; - Ok(None) + None + } + power::IdleAction::Recheck(delay) => Some(delay), + }; + Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }) + } + + /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A + /// refused control (recovery in progress) is retried on a later check. + fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { + const RETRY_NS: u64 = 1_000_000; + let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); + for _ in 0..2 { + match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + power::HoldAction::None => return None, + power::HoldAction::Recheck(delay) => return Some(delay), + power::HoldAction::Assert => { + if self.set_idle_control(false).is_err() { + return None; + } + self.hold.asserted(); + } + power::HoldAction::Release => { + if self.set_idle_control(true).is_err() { + return Some(RETRY_NS); + } + self.hold.released(); + return None; + } } - power::IdleAction::Recheck(delay) => Ok(Some(delay)), } + None + } + + fn set_idle_control(&mut self, enabled: bool) -> Result { + power::Control::set_idle(self, enabled) } } @@ -610,6 +762,11 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, + /// Layout of renders' deflake buffers for this GPU's cores. + deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -742,12 +899,20 @@ impl Shared { } fn drain_reclaims(self: &Arc) { + let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); + reclaimed = true; + } + if reclaimed { + // Context release returns its execution root off-lock. The next + // device pass can now detach idle compute aliases and their QoS + // leases, without depending on a new client's submission/event. + self.queue_events(); } } @@ -826,6 +991,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _capacity: Pin>>, } impl Gpu { @@ -836,6 +1002,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -860,12 +1027,28 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + if cfg + .core_layout + .cores + .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + { + dev_err!( + dev.as_ref(), + "Unsupported GPU core layout: {} clusters with {} cores\n", + id.num_clusters, + active_cores + ); + return Err(ENODEV); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant, + operand_state: &cfg.core_layout.operand_state, + deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -884,6 +1067,11 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", capacity::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -892,6 +1080,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + _capacity: capacity, }, GFP_KERNEL, )?; @@ -901,6 +1090,42 @@ impl Gpu { Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -932,6 +1157,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -948,11 +1174,20 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -964,9 +1199,17 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + hold: power::Hold::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: shared.variant, + queues <- runtime::Registry::new( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + ), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -977,45 +1220,95 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, + shared.clusters, + shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; + } + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if shared.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1081,12 +1374,20 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; - params.gpu_generation = hw::GpuGen::G17 as u32; + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + let variant = self.shared.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1115,9 +1416,18 @@ impl gpu::Gpu for Gpu { true } + /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not + /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 + /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads + /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case + /// failed; all pass with soft faults off). + fn supports_soft_faults(&self) -> bool { + false + } + fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1156,12 +1466,26 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.shared.variant, + )?; let policy = scheduling_policy(priority)?; + let admission_started = now_ns(); let (backend, slot) = loop { match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, + Err(error) if error == ENOSPC => { + // Closed logical queues can still own execution/QoS IDs + // until the scheduler release is consumed and reclaimed. + // Backend construction has rolled back all temporary leases. + if now_ns().saturating_sub(admission_started) + >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } result => break result?, } }; @@ -1185,27 +1509,28 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { + ) -> Result { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; let mut addresses = firmware.uat.geometry().kernel_range(); addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( + crate::timestamp::Buffer::firmware(bo.map_range_into_range( firmware.uat.kernel_vm(), range, addresses, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so -/// MEDIUM arrives as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH +/// arrives as 1, MEDIUM as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { + 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae4..f1b22ad5d73310 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,7 +78,17 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + Placement::In(range, align) => { + let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start + && range.end <= crate::hw::t8140::dynamic::LOWER.end { + super::capacity::Site::LowerVa + } else { + super::capacity::Site::UpperVa + }; + this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + if error == ENOSPC { super::capacity::refused(site) } else { error } + })? + } }; this.mapping = Some(mapping); Ok(this) @@ -113,6 +123,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -159,6 +172,20 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) + } + + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -341,6 +368,76 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = (if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + ) + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + ) + }).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) + } else { error })?; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + ).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) + } else { error })?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware @@ -577,3 +674,16 @@ impl Pool { } } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let inner = self.inner.lock(); + for family in 0..POOL_FAMILIES { + let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; + writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, + inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), + entries.iter().flatten().filter(|entry| entry.poisoned).count())?; + } + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 96b363bc27447e..7fef15084c50a0 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,9 +3,19 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. +use core::sync::atomic::AtomicU32; + pub(super) const FULL_EFFORT: u32 = 0x10000; const IDLE_HOLD_NS: u64 = 16_000_000; +/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after +/// the last job. Not measured on other firmware; at or below it the hold stays off. +pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; + +/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU +/// cores powered until the GPU has been idle this long ([`Hold`]). +pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -17,6 +27,7 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, + submissions: u64, } impl Effort { @@ -25,12 +36,19 @@ impl Effort { streaming: false, idle_since: None, applied: 0, + submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; + self.submissions = self.submissions.wrapping_add(1); + } + + /// Count of published submissions, for [`Hold::step`]. + pub(super) fn submissions(&self) -> u64 { + self.submissions } pub(super) fn applied(&self) -> u32 { @@ -62,6 +80,83 @@ impl Effort { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum HoldAction { + None, + /// Send the keep-powered idle policy. + Assert, + /// Send the power-off idle policy again. + Release, + Recheck(u64), +} + +/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job +/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer +/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off +/// policy once no work has been submitted or in flight for that long. Checked from the event +/// worker, which runs on every completion, well inside the firmware's 2 ms. +pub(super) struct Hold { + active: bool, + seen: u64, + since: Option, +} + +impl Hold { + pub(super) const fn new() -> Self { + Self { + active: false, + seen: 0, + since: None, + } + } + + pub(super) fn active(&self) -> bool { + self.active + } + + pub(super) fn step( + &mut self, + now_ns: u64, + in_flight: bool, + submissions: u64, + hold_ms: u32, + ) -> HoldAction { + let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; + let fresh = submissions != self.seen; + self.seen = submissions; + if in_flight || fresh { + self.since = None; + } + if !self.active { + return if enabled && (in_flight || fresh) { + HoldAction::Assert + } else { + HoldAction::None + }; + } + if in_flight { + return HoldAction::None; + } + let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; + let since = *self.since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= hold_ns { + HoldAction::Release + } else { + HoldAction::Recheck(hold_ns - idle) + } + } + + pub(super) fn asserted(&mut self) { + self.active = true; + } + + pub(super) fn released(&mut self) { + self.active = false; + self.since = None; + } +} + /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -137,3 +232,31 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] +mod tests { + use super::*; + + #[test] + fn holds_until_idle_for_the_delay_then_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); + // A new submission restarts the idle period. + assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + + #[test] + fn the_default_delay_never_holds_and_lowering_it_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); + assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 8563beda979e66..0297c619b4f933 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -20,6 +20,34 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; +/// One ready packet's monotonic compute-admission budget. The callback checks +/// packet cancellation, logical close, VM status and device loss even when no +/// capacity/recovery event arrives. Wakes and nested constructors never renew it. +pub(super) struct Admission<'a> { + deadline: u64, + check: &'a dyn Fn() -> Result, +} + +impl<'a> Admission<'a> { + pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { + Self { deadline, check } + } + + pub(super) fn check(&self) -> Result { + (self.check)()?; + if super::now_ns() >= self.deadline { return Err(EBUSY); } + Ok(()) + } + + /// Ignore condvar remaining-time reports: only the monotonic deadline counts. + pub(super) fn slice_ms(&self) -> Result { + self.check()?; + let remaining = self.deadline.saturating_sub(super::now_ns()); + if remaining == 0 { return Err(EBUSY); } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) + } +} + #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -92,6 +120,27 @@ impl Gate { } } + /// Packet admission may wait for recovery only within its original budget. + /// Called with no device mutex or existing preparation lease. + pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let _ = self.changed.wait_timeout( + &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); + } + } + fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -129,6 +178,17 @@ impl Gate { self.changed.notify_all(); } + /// Runtime failure/recovery must not join an indefinitely stalled builder. + /// Timeout retains every graph and reservation; it never authorizes freeing. + pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + if super::now_ns() >= deadline { return Err(ETIMEDOUT); } + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); + } + Ok(()) + } + /// Join builders with no device, queue or mapping lock held. pub(super) fn wait_drained(&self) { let mut wait = self.wait.lock(); @@ -184,3 +244,96 @@ impl Drop for Lease { } } } + +#[kunit_tests(asahi_g17_growth_gate)] +mod growth_gate_tests { + use super::*; + #[test] + fn recovery_can_join_between_growth_admissions() { + let gate = Gate::new().unwrap(); + let outer = gate.try_enter().unwrap(); + // The growth handoff must drop this lease before enter can wait. + drop(outer); + gate.close(); + assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); + gate.wait_drained(); + assert!(gate.reopen()); + let growth = gate.enter(false).unwrap(); + assert!(growth.is_current()); + drop(growth); + } +} + +#[kunit_tests(asahi_g17_compute_deadline)] +mod compute_deadline_tests { + use super::*; + + #[test] + fn spent_budget_refuses_even_an_open_gate() { + let gate = Gate::new().unwrap(); + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn closed_gate_expires_without_any_recovery_wake() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(super::super::now_ns() - start < 500_000_000); + gate.wait_drained(); + assert!(gate.reopen()); + } + + #[test] + fn cancellation_without_capacity_or_recovery_events() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || { + if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }; + let admission = Admission::new(start + 2_000_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn recovery_and_binding_retry_churn_cannot_renew_budget() { + let gate = Gate::new().unwrap(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + loop { + // Re-entry models a channel taken between admission/publication. + // Recovery joins each dropped lease, then wakes the same budget. + gate.close(); + gate.changed.notify_all(); + gate.wait_drained(); + assert!(gate.reopen()); + match gate.enter_until(&admission) { + Ok(lease) => drop(lease), + Err(error) => { assert_eq!(error, EBUSY); break; } + } + } + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn packet_status_and_cancel_precede_available_capacity() { + let gate = Gate::new().unwrap(); + for error in [ECANCELED, EIO] { + let check = || Err(error); + let admission = Admission::new(u64::MAX, &check); + assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f6..ebdc79d2b2dd3d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -47,12 +47,25 @@ struct InFlight { const QUARANTINE_WAIT_MS: u32 = 1000; +/// A wake never renews admission's budget. Short observations also notice a +/// packet cancellation that has no corresponding pair-release notification. +fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { + let remaining = deadline_ns.saturating_sub(now_ns); + if remaining == 0 { + return Err(EBUSY); + } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) +} + struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, + /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail + /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. + spared: u64, kill_hold: u64, released: u64, epoch: u64, @@ -68,6 +81,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, + spared: 0, kill_hold: 0, released: 0, epoch: 0, @@ -92,7 +106,7 @@ impl State { & Self::MASK == 0 => { - Err(ENOSPC) + Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) } _ => Ok(()), } @@ -115,18 +129,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,9 +191,52 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; } } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; + } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true + } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { return None; @@ -188,9 +247,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -283,23 +364,33 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, } impl Pool { - pub(crate) fn new() -> Result> { + pub(crate) fn new(max_queues: u32) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, }), GFP_KERNEL, ) } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); + if count >= self.max_queues { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( count, @@ -316,16 +407,22 @@ impl Pool { self: &Arc, owner: u64, request: Option>, + deadline_ns: u64, + check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let started = crate::g17::now_ns(); + let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); + let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); let mut expired = false; let mut ticket = None; let result = (|| loop { + check()?; + let now = crate::g17::now_ns(); + let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -337,20 +434,26 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { - return Err(EIO); + if now >= quarantine_deadline { + // Behind an innocent pair's quarantine (another VM wedged the GPU) only this + // publication fails, not the VM: it recovers once the pair drains. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + return Err(if spared { EBUSY } else { EIO }); + } + let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { + return Err(ERESTARTSYS); } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; continue; } state.refusal(owner)?; + expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -376,18 +479,17 @@ impl Pool { } } } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { + } + // Keep the established hazard wait/fallback rule, but cap every + // sleep with the caller's original deadline and cancellation poll. + let slice = if plan != Plan::Any { + remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) + } else { + remaining_ms + }; + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { return Err(ERESTARTSYS); } })(); @@ -420,8 +522,33 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -446,12 +573,82 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; + state.spared &= !released; state.kill_hold = 0; drop(state); self.released.notify_all(); } } +#[kunit_tests(asahi_g17_admission_accounting)] +mod accounting_tests { + use super::*; + + #[test] + fn every_logical_owner_fits_without_speculative_spares() { + let mut state = State::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot = state.acquire(owner).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); + state.release(slot); + } + assert_eq!(state.active, 0); + assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); + } + + #[test] + fn close_does_not_reuse_an_active_lease() { + let mut state = State::new(); + let slot = state.acquire(1).unwrap(); + state.released |= 1u64 << slot; + assert!(!state.reassign_unused_spare(slot, Some(1), None)); + state.release(slot); + assert!(state.reassign_unused_spare(slot, Some(1), None)); + assert_eq!(state.owners[usize::from(slot)], 0); + assert_eq!(state.refs[usize::from(slot)], 0); + assert_eq!(state.active & (1u64 << slot), 0); + } + + #[test] + fn warming_quarantine_and_exit_hold_exclude_reassignment() { + let mut state = State::new(); + let primary = state.acquire(1).unwrap(); + // A retained legacy sibling still obeys all quarantine/kill guards. + state.owners[2] = 1; + let spare = state.borrow_warm_sibling(1, primary).unwrap(); + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.release(spare); + state.quarantined |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.quarantined = 0; + state.kill_hold |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.kill_hold = 0; + assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); + assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + state.release(primary); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + } + + #[test] + fn logical_queue_count_returns_on_drop() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..MAX_EXECUTION_CONTEXTS { + leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); + assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); + drop(leases.pop()); + let replacement = pool.reserve_queue().unwrap(); + drop(leases); + assert_eq!(pool.queues.load(Ordering::Acquire), 1); + drop(replacement); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } +} + /// Counted once per userspace logical queue, independently of installed graphs. pub(crate) struct QueueSlot { pool: Arc, @@ -468,17 +665,40 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } - pub(crate) fn quarantine(&self) { - self.pool.state.lock().quarantined |= 1u64 << self.slot; + /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine + /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. + pub(crate) fn quarantine(&self, spared: bool) { + let bit = 1u64 << self.slot; + let mut state = self.pool.state.lock(); + if !spared { + state.spared &= !bit; + } else if state.quarantined & bit == 0 { + state.spared |= bit; + } + state.quarantined |= bit; + drop(state); self.pool.released.notify_all(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - self.pool.state.lock().quarantined &= !(1u64 << self.slot); + let mut state = self.pool.state.lock(); + state.quarantined &= !(1u64 << self.slot); + state.spared &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { @@ -539,3 +759,169 @@ impl Pool { self.released.notify_all(); } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; + let state = self.state.lock(); + let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { + bits | if *owner != 0 { 1u64 << slot } else { 0 } + }) & State::MASK; + let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; + writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", + used.count_ones(), State::MASK.count_ones(), retained.count_ones(), + (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), + (state.kill_hold & State::MASK).count_ones()) + } +} + +#[kunit_tests(asahi_g17_fault_admission)] +mod fault_capacity_tests { + use super::*; + + #[test] + fn a_guilty_failure_makes_a_spared_quarantine_guilty() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + lease.quarantine(true); + assert_ne!(pool.state.lock().spared & bit, 0); + lease.quarantine(false); + assert_eq!(pool.state.lock().spared & bit, 0); + // Once guilty, a later spared failure does not make it spared again. + lease.quarantine(true); + assert_eq!(pool.state.lock().spared & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + pool.release_owner(1); + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.quarantined | state.spared, 0); + } + + #[test] + fn closed_faulting_clients_return_leases_after_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + for owner in 1..=512 { + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + lease.quarantine(false); + pool.release_owner(owner); + let bit = 1u64 << lease.slot(); + assert_ne!(pool.state.lock().active & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + // The runtime, not an error fence, supplies this exact drain witness. + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.active, 0); + assert_eq!(state.quarantined, 0); + assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } + } + + #[test] + fn published_exit_kill_holds_survive_close_and_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); + // A failed (unpublished) control may roll back its hold. Published kills + // keep the existing processor-stop rule; this test must not weaken it. + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().kill_hold, 0); + } +} + +impl Pool { + pub(crate) fn owner_active(&self, owner: u64) -> bool { + let state = self.state.lock(); + state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) + } +} + +#[kunit_tests(asahi_g17_shared_render_depth)] +mod shared_depth_tests { + use super::*; + #[test] + fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { + let mut state = State::new(); + let first = state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } + assert_eq!(state.acquire(1), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + state.release(first); + assert_eq!(state.acquire(1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_admission_deadline)] +mod deadline_tests { + use super::*; + + #[test] + fn all_pairs_busy_no_progress_expires_without_release() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + // No pair has room, including the requesting owner's retained pair. + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + assert!(matches!(result, Err(error) if error == EBUSY)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn busy_pair_observes_cancellation_without_a_release_event() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }); + assert!(matches!(result, Err(error) if error == ECANCELED)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn cancellation_precedes_available_capacity() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + Err(error) if error == ECANCELED)); + assert_eq!(pool.state.lock().active, 0); + } + + #[test] + fn unrelated_wake_storm_never_renews_absolute_deadline() { + let deadline = 2_000_000_000; + // Each wake is allowed to report the full jiffy budget remaining. Only + // monotonic time, not that report or a wake count, bounds admission. + for now in (0..deadline).step_by(100_000) { + assert!(wait_slice_ms(deadline, now).unwrap() <= 10); + } + assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); + assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb7609..713809907dd4bf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,7 +9,12 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{ + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, + }, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -113,17 +118,26 @@ struct Graph { records: KernelObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new( + alloc: &Allocator<'_>, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; @@ -141,7 +155,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; @@ -160,6 +174,8 @@ impl Graph { records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, + operand_state, }) } @@ -221,7 +237,7 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) @@ -258,7 +274,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); + return Err(EAGAIN); } } Ok(()) @@ -276,7 +292,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(POINTERS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -389,11 +407,21 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous { - binding: Option, - context: Arc, +struct Previous> { + binding: Option, + context: C, publication: Option<(u64, u64)>, } +impl Previous { + /// The exact drain witness covers the replacement publication even when + /// its public fence failed. An unpublished replacement still needs rollback. + fn retire_witnessed(previous: &mut Option) { + if previous.as_ref().is_some_and(|p| p.publication.is_some()) { + drop(previous.take()); + } + } +} + /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -453,12 +481,21 @@ pub(crate) enum PoolPhase { Vacant, } +/// Builder ownership of one physical release transaction. The preparation +/// lease pins this registry until abort/commit; owner close may remove lookups. +#[derive(Clone, Copy)] +pub(crate) struct Growth { + qid: u8, + phase: PoolPhase, +} + pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -471,6 +508,8 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, + /// This binding has never crossed the successor visibility boundary. + unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -509,9 +548,11 @@ impl Queue { backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -520,6 +561,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -527,6 +569,7 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, + unpublished_binding: true, kick, pool, graph, @@ -566,6 +609,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -574,18 +632,16 @@ impl Queue { pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - pub(crate) fn room(&self) -> bool { - !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT - } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: Arc::as_ptr(active.packet.completion.status()) as usize, + vm: active.packet.context.vm().id(), published: active.published, finished: active.observed.is_some(), + failed: self.quarantined, }) } @@ -626,6 +682,25 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + + pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { + self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + } + + /// A closed owner can still retain its last commands or their completion + /// stamps. Admission may wait for retirement, without treating that close + /// as permission to reuse any firmware-owned backing. + pub(crate) fn release_settling(&self) -> bool { + // release_owner keeps the old owner while its last commands retire. + // Waiting only once owner is None misses precisely that interval. + release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -780,9 +855,7 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = front - .observed_at - .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); + let record_aged = retirement::record_aged(front.observed_at, now_ns()); let timestamps = retirement::poll( &ticket, front.observed, @@ -829,6 +902,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { @@ -929,6 +1003,10 @@ impl Queue { self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; + // The stamp and fully consumed ring proved the replacement retired. + // Keeping its predecessor only on the fault path prevents recycling + // forever: neither another retirement nor an exit kill can release it. + Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -937,15 +1015,9 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } if self.released { self.owner = None; + self.released_at = now_ns(); } } Ok(()) @@ -1027,6 +1099,102 @@ impl Queue { } } +/// Validate the complete unpublished item budget before changing descriptors, +/// work counts or queue producers. Leave one item empty across ring wrap. +fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { + if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); + return Err(EAGAIN); + } + Ok(()) +} + +/// Retained commands are capacity, not a failed VM. Nothing has been written +/// when this check runs, so the outer publisher can wait for retirement under +/// its original admission deadline, just as it does for a full firmware ring. +fn retained_capacity( + quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, +) -> Result { + if quarantined || released { + return Err(EBUSY); + } + if active >= IN_FLIGHT || (replays != 0 && !replay_front) { + return Err(EAGAIN); + } + Ok(()) +} + +/// Logical close promises a future healthy handoff, not immediate reuse. +/// Exact retirement still owns the transition. +fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { + released && !quarantined && (active != 0 || retiring) +} + +#[kunit_tests(asahi_g17_compute_admission)] +mod admission_tests { + use super::*; + + #[test] + fn retained_ring_pressure_waits_without_discarding_work() { + let mut active = Fifo::new(); + for sequence in 0..IN_FLIGHT { + active.push(sequence).unwrap(); + } + assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!(active.front(), Some(&0)); + assert_eq!(active.len, IN_FLIGHT); + assert_eq!(active.pop(), Some(0)); + assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!(active.front(), Some(&1)); + } + + #[test] + fn replay_front_and_terminal_ownership_remain_distinct() { + assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); + assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); + assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + for (quarantined, released) in [(true, false), (false, true), (true, true)] { + assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + } + } + + #[test] + fn item_refusal_is_retryable_and_preserves_the_empty_slot() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + } + + #[test] + fn bad_item_cursors_are_not_backpressure() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); + assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + } + + #[test] + fn closed_owner_waits_for_its_last_command() { + assert!(release_pending(true, false, 1, false)); + assert!(release_pending(true, false, 0, true)); + assert!(!release_pending(true, false, 0, false)); + } + + #[test] + fn live_or_quarantined_owners_do_not_promise_capacity() { + assert!(!release_pending(false, false, 1, true)); + assert!(!release_pending(true, true, 1, true)); + } +} + impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1107,6 +1275,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1134,6 +1303,13 @@ impl Queue { } self.retirement_ready = true; } + // Rebinding an already detached queue has no Previous. If its + // first command is rejected or cancelled before publication, the + // empty active FIFO is still the original idle witness. No future + // completion can turn retirement_ready back on for this owner. + if self.previous.is_none() { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1259,6 +1435,7 @@ impl Queue { ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); + self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1283,7 +1460,7 @@ impl Queue { let pages = self.pool.take_pages(); let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; + self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1298,12 +1475,11 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } + if !self.can_rearm() { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1314,11 +1490,7 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { - if blocks < self.growth_blocks { return Err(EAGAIN); } - self.growth_blocks = blocks; - return Ok(false); - } + if self.growing { return Err(EAGAIN); } if self.quarantined || self.released || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) @@ -1327,7 +1499,8 @@ impl Queue { let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - } else { self.pool_phase = PoolPhase::Vacant; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) @@ -1344,16 +1517,35 @@ impl Queue { return Err(EAGAIN); } } + if alias.is_none() { return Err(EINVAL); } + let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - let old_alias = binding.pool_alias.take(); - let old_pages = self.pool.take_pages(); - self.pool.rearm(pages.take().ok_or(EINVAL)?)?; - init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; - binding.pool_alias = alias.take(); + // No ownership moves before every fallible descriptor/header operation. + // The old release is consumed; no queue can use this row until commit. + init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + self.pool.replace_pages(pages)?; + let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + } + + pub(crate) fn growth_token(&self) -> Option { + self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + } + + /// Transfer an abandoned builder's release back to the event worker. Do not + /// revoke the published release or free backing here: vacancy still requires + /// its exact control-consumer witness. A different transaction is untouched. + pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { + if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { + return Ok(None); + } + self.growing = false; + self.growth_blocks = 0; + self.release_owner().map(Some) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -1436,14 +1628,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(POINTERS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(POINTERS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1535,7 +1730,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1618,6 +1813,7 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); + item_capacity(self.graph.cursors()?, entry_signal)?; let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1640,6 +1836,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -1726,16 +1923,6 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); - } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1743,6 +1930,7 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1779,6 +1967,7 @@ impl Queue { Err(error) })?; visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1898,11 +2087,8 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } + retained_capacity(self.quarantined, self.released, self.active.len, + self.replays.len, replay_front)?; if !packet.context.is_current() || !self .binding @@ -1969,7 +2155,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -1982,3 +2168,153 @@ impl Queue { self.terminal.take() } } + +impl Queue { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", + self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), + u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), + u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; + match self.graph.cursors() { + Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + _ => writeln!(f, "items_unavailable=1"), + }?; + writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), + u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + if let Some(binding) = self.binding.as_ref() { + binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + } + if let Some(previous) = self.previous.as_ref() { + previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + } + Ok(()) + } +} + +#[kunit_tests(asahi_g17_fault_ownership)] +mod fault_ownership_tests { + use super::*; + use core::sync::atomic::AtomicUsize; + + struct Lease(Arc); + impl Lease { + fn new(used: &Arc) -> Self { + used.fetch_add(1, Ordering::Relaxed); + Self(used.clone()) + } + } + impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + + // This tests the production predecessor ownership operation without firmware + // storage. Runtime tests must separately establish its drain/kill witness. + fn faulting_clients(publication: Option<(u64, u64)>) { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + for _ in 0..512 { + let mut previous = Some(Previous { + binding: Some(Lease::new(&used)), + context: Lease::new(&used), + publication, + }); + assert_eq!(used.load(Ordering::Relaxed), 2); + Previous::retire_witnessed(&mut previous); + assert_eq!(previous.is_none(), publication.is_some()); + if publication.is_none() { + assert_eq!(used.load(Ordering::Relaxed), 2); + // An unpublished replacement returns its original owner on close. + drop(previous.take()); + } + assert_eq!(used.load(Ordering::Relaxed), 0); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn faulted_published_replacements_return_to_baseline() { + faulting_clients(Some((1, 7))); + } + + #[test] + fn unpublished_replacements_survive_until_rollback() { + faulting_clients(None); + } + + #[test] + fn same_vm_replacement_without_binding_returns_its_context() { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + let mut previous = Some(Previous:: { + binding: None, + context: Lease::new(&used), + publication: Some((9, 3)), + }); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } +} + +impl Queue { + /// Offer an open client's idle physical channel to another ready job. + /// Logical queue/context ownership is unchanged. The existing binding is + /// retained through the successor's exact replacement witness. + pub(crate) fn share_idle(&mut self) -> Result { + if !self.installed || self.owner.is_none() || !self.retirement_ready + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.previous.is_some() || self.pool_phase != PoolPhase::Populated + { + return Ok(false); + } + self.release_owner()?; + Ok(self.reusable()) + } +} + +impl Queue { + pub(crate) fn can_rearm(&self) -> bool { + self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + } +} + +impl Queue { + /// A failed first attempt has no future completion to undo its binding. + /// Return the replaced aliases to the caller for off-lock destruction. The + /// caller validates the current preparation epoch and cancels only a true + /// result's never-published QID; installed rings and identities stay intact. + pub(crate) fn rollback_unpublished( + &mut self, owner: u64, context: &Arc, + ) -> Result<(bool, Option)> { + if self.owner != Some(owner) || !self.unpublished_binding + || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.retire_pending || self.failure_status_pending + || self.pool_phase != PoolPhase::Populated + || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + { + return Ok((false, None)); + } + if !self.installed { return Ok((true, None)); } + let mut retired = None; + if let Some(previous) = self.previous.as_ref() { + self.graph.set_owner(&previous.context)?; + } + if let Some(mut previous) = self.previous.take() { + if let Some(binding) = previous.binding.take() { + retired = self.binding.replace(binding); + } else { + self.binding.as_mut().ok_or(EIO)?.context = previous.context; + } + } + self.unpublished_binding = false; + self.retirement_ready = true; + self.released = true; + self.owner = None; + self.released_at = now_ns(); + Ok((false, retired)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 187fd27f8e78e6..d7386e335e1671 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -220,6 +220,9 @@ impl Queue { ))?; drop(self.active.pop()); } + // The acknowledged kill and command settlement replace the ordinary + // drain witness, including a failed first publication after rebinding. + super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421f..e938ec566dbacb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -66,21 +66,37 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. The firmware stores the stamp after the cache -/// maintenance that follows the record; it occasionally never stores it for a command, most -/// visibly while a render of the same VM is running. -pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; +/// before it retires on the record alone. +/// +/// The firmware stores the stamp after the cache maintenance that follows the record, so the +/// stamp, not the record, proves that the command's writes are complete and that its mappings +/// may be released. While other commands of the same submit ioctl are still running, the stamp +/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms +/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as +/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail +/// its VM. +pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; + +// A command that waits out the fallback must still retire well inside the scheduler timeout. +const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); + +/// Whether a completion record latched at `observed_at` (host time, ns) has waited out +/// [`STAMP_FALLBACK_NS`] by `now`. +pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { + observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) +} static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -/// Counts a command that retired on its record without its completion stamp; logs the first and -/// every 64th. +/// Counts a command that retired on its record because its completion stamp never arrived; +/// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, + STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -90,7 +106,12 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// must be consumed. +#[expect( + clippy::too_many_arguments, + reason = "Retirement validates the ticket against independent firmware observations." +)] pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -208,7 +229,8 @@ impl Batch { if descriptor == 0 || record.read(7)? == 0 { return Ok(()); } - let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -240,3 +262,143 @@ impl Batch { Ok(request.observed.take()) } } + +#[kunit_tests(asahi_g17_compute_stamp_wait)] +mod tests { + use super::*; + + const KICK: u64 = 0x12_3456_7801; + + fn ticket() -> Ticket { + Ticket { + owner: 1, + submission: 1, + context_id: 1, + qid: 28, + descriptor: 0x1000, + work_node: 0x2000, + kick: KICK, + ordinal: 1, + item_producer: 4, + item_count: 64, + epoch: 1, + } + } + + fn consumed() -> Cursors { + Cursors { + producer: 4, + consumer: 4, + count: 64, + } + } + + fn own(kick: u64) -> u32 { + (kick as u32).wrapping_shl(8) + } + + #[test] + fn stamp_covers_its_kick_and_earlier_ones() { + assert!(stamp_covers(own(KICK), KICK)); + assert!(stamp_covers(own(KICK + 1), KICK)); + assert!(!stamp_covers(own(KICK - 1), KICK)); + assert!(!stamp_covers(own(KICK) | 1, KICK)); + assert!(stamp_covers(0, 0xff_ffff)); + } + + #[test] + fn a_late_stamp_is_waited_for() { + // Under games on J700 the stamp lands up to about 4 ms after the record while the + // rest of the command's ioctl runs: the command keeps its mappings until then. + let record = Some(Observation { timestamps: [10, 20] }); + let at = 1_000_000; + assert!(!record_aged(Some(at), at + 4_000_000)); + assert!(!record_aged(Some(at), at + 10_000_000)); + let waiting = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + 10_000_000), + ); + assert_eq!(waiting, Ok(None)); + let stamped = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(stamped, Ok(Some([10, 20]))); + } + + #[test] + fn a_later_kick_stamp_retires_it() { + let record = Some(Observation { timestamps: [10, 20] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK + 3), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(result, Ok(Some([10, 20]))); + } + + #[test] + fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { + let record = Some(Observation { timestamps: [0x1111, 0x2222] }); + let at = 1_000_000; + assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); + assert!(!record_aged(None, u64::MAX)); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + STAMP_FALLBACK_NS), + ); + // The record's own timestamps are published, never synthesized ones. + assert_eq!(result, Ok(Some([0x1111, 0x2222]))); + } + + #[test] + fn an_aged_record_still_waits_for_consumed_items() { + let record = Some(Observation { timestamps: [10, 20] }); + let busy = Cursors { + producer: 4, + consumer: 2, + count: 64, + }; + let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); + assert_eq!(result, Ok(None)); + } + + #[test] + fn a_zero_duration_record_fails() { + let record = Some(Observation { timestamps: [5, 5] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::Allowed, + 0, + None, + true, + ); + assert_eq!(result, Err(ENODATA)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8e..6a9b15789339fe 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -56,10 +56,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +77,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,6 +92,7 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], @@ -94,6 +102,8 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } +type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); + impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -107,6 +117,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -119,6 +130,7 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), @@ -128,7 +140,8 @@ impl Queue { } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Admission is checked even when this queue already owns a scheduler. + // Validate every request, including an existing scheduler. Physical + // admission happens after its DRM dependencies, in backend publication. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -156,6 +169,10 @@ impl Queue { }); } + #[expect( + clippy::too_many_arguments, + reason = "Enqueue keeps validated command, ordering, and destination owners explicit." + )] fn enqueue( &mut self, id: u64, @@ -270,6 +287,13 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { + // An input that already signalled without error orders nothing and has no status + // left to consume (like a completed frontier producer above). Keeping it would + // send the render through the scheduler thread instead of inline publication: + // most compositor and client renders carry such already-signalled buffer fences. + if job::fence_status(fence) > 0 { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } @@ -307,7 +331,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { + ) -> Result { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -320,6 +344,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { @@ -384,7 +409,10 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes, objects)?; + let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + } let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e402..fb6c77fbdf04f2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -87,6 +87,8 @@ pub(crate) trait Host { fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; + /// The next publication number, shared with compute (timeout ordering). + fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -120,6 +122,8 @@ struct Prepared { mcache: Option, } struct Active { + /// Publication number, shared with compute (crate::g17::timeout). + published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -152,8 +156,11 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -168,14 +175,28 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// A retired, never-executed group awaiting republication on this pair. - replay: Option>, + /// Retired, never-executed groups awaiting republication on this pair, in + /// their original order. Nothing else is prepared on the pair before them. + replay: Tracker>, + /// When the oldest parked group was parked or the last one republished. + replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "The packet constructor spells out the paired render firmware inputs." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -186,7 +207,9 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -211,10 +234,11 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -226,9 +250,10 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -239,8 +264,11 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, + deflake, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -251,8 +279,11 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -280,20 +311,25 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.is_some() + self.active.len() != 0 || self.replay.len() != 0 } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_stalled(&self, now: u64) -> bool { + self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.clone() + self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay = None; + self.replay.pop(); + // Progress: the expiry restarts for the next parked group, if any. + self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -315,6 +351,7 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() + || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { @@ -334,14 +371,17 @@ impl Pair { let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -371,6 +411,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -402,11 +443,18 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -503,9 +551,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -517,6 +573,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -619,6 +682,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -628,6 +692,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -649,6 +714,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -658,6 +724,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -694,8 +761,10 @@ impl Pair { let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; + let published = host.next_publication()?; self.active .push(Active { + published, qos_completed: false, flist_completed: false, outer_published: false, @@ -730,9 +799,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -747,7 +813,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -781,6 +852,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -924,6 +1000,22 @@ impl Pair { .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) }) } + /// The oldest published group still running, for cross-VM timeout attribution + /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains + /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. + pub(crate) fn timeout_head(&self) -> Option { + self.active + .iter() + .filter(|active| active.outer_published) + .find(|active| active.drained_result.is_none()) + .map(|active| crate::g17::timeout::Head { + vm: active.packet.context.vm().id(), + published: active.published, + finished: false, + failed: self.quarantined, + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -934,7 +1026,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -955,7 +1047,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -1084,161 +1176,218 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - loop { - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { return Err(EIO); } - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { - return Err(EIO); - } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); } - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(true) } - /// The front group was retired and unlinked by firmware without executing - /// (job list empty, stamps cover it, no recovery, fragment end never - /// written). Nothing of this group reached its attachments, so the same - /// command is published again instead of failing the owner's VM. The - /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, - /// and only for a lone group whose completion still accepts a replay. + /// Every retained group was retired and unlinked by firmware without + /// executing (job list empty, stamps cover it, no recovery, fragment end + /// never written): after another VM's recovery the firmware drops an + /// innocent pair's queued groups this way (seen on 25G83 and 26A428). + /// Nothing of them reached their attachments, so the same commands are + /// published again, in order, instead of failing their owners. Only the + /// whole set is replayed; a replayed prefix would run after a live + /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS + /// apart (worker polling); the render timeout, long after publication, + /// takes it once as its retirement does. A single sample can still + /// straddle a firmware update of the fragment end record; the cost is + /// that a group which did execute runs a second time (blending twice). fn replay_abandoned( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, ) -> Result { use retirement::Memory; - if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } - let front = self.active.front().ok_or(EIO)?; - if !front.outer_published || front.failed { - return Ok(false); + use crate::g17::replay::{self, Decision, Group}; + let qids = self.qids(); + let memory = &self.memory; + let mut failure = None; + let groups = self.active.iter().map(|active| { + let innocent = + active.outer_published && !active.failed && !active.packet.completion.guilty(); + let blamed = active.packet.completion.blamed(); + let exhausted = + active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; + // decide() stops at the first group that is not abandoned, an error included. + let abandoned = innocent + && !blamed + && (|| -> Result { + let mut observer = Observer { + memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof; a later poll or the timeout looks again. + Ok(matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + )) + })() + .unwrap_or_else(|error: Error| { + failure = Some(error); + false + }); + Group { + innocent, + blamed, + abandoned, + exhausted, + } + }); + let decision = replay::decide(groups); + if let Some(error) = failure { + return Err(error); } - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(front.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - let abandoned = - retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if !abandoned { - front.abandoned_since = None; - return Ok(false); + match decision { + Decision::Refuse => return Ok(false), + Decision::Wait => { + front.abandoned_since = None; + return Ok(false); + } + Decision::Replay => {} } - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } } - if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_info!( - "render slot {} retired without execution (pass {:?}); replaying\n", + // decide() lets only the front be exhausted. An exhausted front fails, and its + // successors, which consume its output, fail with it rather than run without it. + let mut failing = self.active.front().is_some_and(|front| { + front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS + }); + if failing { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", + self.slot, + self.active.len() + ); + } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replaying\n", self.slot, - snapshot.pass + self.active.len() ); } let qos = self.qos_owners(); let buffer = self.buffer_id(); let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.flist_completed = true; - front.qos_completed = true; - } - let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - if !packet.completion.begin_replay() { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); + // Every group parked below shares this expiry (an early return keeps it). + self.replay_since = Some(now); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + failing = failing || !packet.completion.begin_replay(); + if failing { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + continue; } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - return Ok(true); + packet.reset_publication_for_replay(); + self.replay.push(packet).map_err(|_| EIO)?; } - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.replay = Some(packet); Ok(true) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). ECANCELED leaves the VM usable; any other error - /// is the ordinary device-loss outcome the replay tried to avoid. + /// publication refusal). Its successors cannot run without it and fail with + /// it. ECANCELED leaves the VM usable; any other error is the ordinary + /// device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(packet) = self.replay.take() else { - return Ok(()); - }; - defer(Deferred::Retired(packet.completion.clone(), Err(error))) + self.replay_since = None; + while let Some(packet) = self.replay.pop() { + defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; + } + Ok(()) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1357,22 +1506,51 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - if !self.quarantined && self.active.len() == 1 { - let active = self.active.front().ok_or(EIO)?; - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if active.outer_published - && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? - { + // Groups the firmware dropped unexecuted are published again, with their successors. + if self.replay_abandoned(host, defer, false)? { + return Ok(true); + } + // Fallback when a live or exhausted group prevents an ordered replay: a spared front group + // the firmware retired without executing during another VM's recovery is retired on its + // own even with successors queued, so the pair is not quarantined into a sticky EIO. It + // fails with ECANCELED, which spared work never reports as device loss; consecutive + // abandoned spared successors are retired the same way. + let front_spared = self.active.front().is_some_and(|a| { + Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() + }); + if !self.quarantined && (self.active.len() == 1 || front_spared) { + let mut retired = false; + loop { + let Some(active) = self.active.front() else { + break; + }; + let spared = active.packet.completion.spared(); + if retired && !spared { + break; + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof, as in replay_abandoned(): the group stays retained + // (quarantined with the pair if nothing was retired) instead of the error + // stopping the runtime for every client. + if !active.outer_published + || !matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + break; + } let qos = self.qos_owners(); let buffer = self.buffer_id(); + let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1384,19 +1562,30 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !active.packet.completion.spared() { + if !active.failed && !spared { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } + pr_warn!( + "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", + active.ticket.ordinal, + left + ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(EIO), + Err(if spared { ECANCELED } else { EIO }), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + retired = true; + if !spared { + break; + } + } + if retired { return Ok(true); } } @@ -1424,7 +1613,8 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - if let Some(packet) = self.replay.take() { + self.replay_since = None; + while let Some(packet) = self.replay.pop() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -1578,6 +1768,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -1621,6 +1823,10 @@ impl Pair { self.owner = None; } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Binding keeps the render owners and first packet explicit." + )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -1654,7 +1860,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -1670,6 +1876,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -1736,6 +1943,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d135..80ee0b871cc829 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -94,6 +97,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +106,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers)?; Ok(()) })?; } @@ -166,6 +172,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +195,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +217,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -310,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -365,6 +386,10 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Render memory construction keeps each allocation and firmware input explicit." + )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -376,10 +401,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -505,6 +531,9 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { @@ -598,6 +627,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e077..209ce0b8176ecb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,12 +25,19 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -48,11 +55,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -76,12 +92,15 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -96,11 +115,22 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Preparation forwards the render binding inputs without changing their ownership." + )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -125,7 +155,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -141,6 +171,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -190,6 +221,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, @@ -202,7 +234,9 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -213,8 +247,11 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 0fc9c1d9a15dcf..9f4cb05b6bedeb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,21 +73,28 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, @@ -116,7 +124,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = (&self.config as *const QueueConfig).cast::(); + let source = core::ptr::from_ref(&self.config).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index d0a2d566bdc8d3..24aeed6bcd0ec3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -203,8 +203,12 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - let observation = observe(ticket, snapshot)?; - complete_poll(ticket, memory, snapshot, observation) + match observe(ticket, snapshot) { + Ok(observation) => complete_poll(ticket, memory, snapshot, observation), + // The borrowed snapshot can straddle a firmware cursor update like any other sample: + // take the ordinary poll's fresh snapshot and reread instead of failing the client. + Err(_) => poll(ticket, memory), + } } fn complete_poll( diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec61..da90488bfedec9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0)?; } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ba368341d84d26..89f8d33bdb80c0 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,11 +9,12 @@ mod attribution; mod slot; +mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; @@ -99,6 +100,8 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, + timeouts: safety::Storm, + restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -112,8 +115,12 @@ impl State { Self { accepted: 0, pending: None, + timeouts: safety::Storm::new(), + restarts: safety::Storm::new(), } } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -220,9 +227,17 @@ pub(super) fn service( } host.recovery_mut().pending = None; - let cancelled = classify(host, entries, request.blamed, &mut sources); + let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { - host.settle_blamed_render(qid, EIO); + pr_warn!( + "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", + sources.reason, + qid, + if blame { "failing its oldest render" } else { "attribution cleared it; spared" } + ); + if blame { + host.settle_blamed_render(qid, EIO); + } } if cancelled != 0 { host.settle_cancelled_renders(cancelled); @@ -235,7 +250,10 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { +/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed +/// render that had not started while others had (or with no culprit at all) is innocent, and its +/// work is spared like that of every other innocent VM instead of failing with EIO. +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); @@ -245,8 +263,9 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; + let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return 0; + return (0, blame); } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -282,7 +301,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return 0; + return (0, blame); } let spared = queues.classify_work(&|status| { guilty[..count] @@ -295,5 +314,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }) .unwrap_or(Class::Spared) }); - cancelled & !(guilty_qids & !collateral) & spared + (cancelled & !(guilty_qids & !collateral) & spared, blame) } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs new file mode 100644 index 00000000000000..c30e45e22579a9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fail closed on repeated watchdog/restart traffic, even across client loss. + +const WINDOW_NS: u64 = 30_000_000_000; +const LIMIT: u8 = 8; + +pub(super) struct Storm { + start: Option, + count: u8, + stopped: bool, +} +impl Storm { + pub(super) const fn new() -> Self { + Self { start: None, count: 0, stopped: false } + } + pub(super) fn note(&mut self, now: u64) -> bool { + if self.stopped { return true; } + if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { + self.start = Some(now); + self.count = 0; + } + self.count += 1; + self.stopped = self.count >= LIMIT; + self.stopped + } +} + +#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] +mod tests { + use super::*; + use crate::g17::{preparation::Gate, recovery::State}; + use kernel::prelude::*; + + #[test] + fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { + let gate = Gate::new().unwrap(); + let lease = gate.try_enter().unwrap(); + let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); + gate.close(); + for n in 0..64 { + let stopped = { + let mut state = state.lock(); + state.request(0, -1).unwrap(); + assert!(state.pending()); + let stopped = state.timeout_limit(n * 2_000_000_000); + if stopped { gate.fail(); } // production state -> gate order + stopped + }; + assert_eq!(stopped, n >= u64::from(LIMIT - 1)); + } + // Recovery/stop must release state before joining the retained builder. + assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), + Err(error) if error == ETIMEDOUT)); + assert!(!gate.reopen()); // failure never gives back an owned reservation + drop(lease); + gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); + assert!(!gate.reopen()); + } + #[test] + fn successful_handshakes_do_not_reset_a_restart_storm() { + let mut state = State::new(); + for n in 0..64 { + assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); + } + assert!(state.restart_limit(u64::MAX)); + } + #[test] + fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } + for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } + assert!(storm.note(64 * WINDOW_NS)); + for _ in 0..1024 { assert!(storm.note(u64::MAX)); } + } + #[test] + fn replay_dependency_checks_never_wait_for_event_service() { + use crate::g17::dependency::replay_status; + assert_eq!(replay_status(0), Err(EBUSY)); + assert_eq!(replay_status(1), Ok(())); + assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 307534708e922f..bd9d4c96be14f6 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as _, + bindings::MEMREMAP_WB as usize, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs new file mode 100644 index 00000000000000..94daa5ec0dfebc --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Whether a pair's retained render groups, retired by firmware without executing, are +//! published again. Only the whole set is replayed, so no replayed group runs after a live +//! successor, and work a recovery blamed is never run again. + +/// One retained group, front first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + /// Published, not failed and not classified guilty by a recovery. + pub(crate) innocent: bool, + /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): + /// it may have caused the fault. + pub(crate) blamed: bool, + /// Carries the strict abandonment witness. + pub(crate) abandoned: bool, + /// Already republished the maximum number of times. + pub(crate) exhausted: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Replay, + /// A group may still execute: look again later. + Wait, + /// Never replay this set; the render timeout settles it. + Refuse, +} + +/// Groups are read in order and the first one that decides stops the walk, so a caller can +/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its +/// successors, which consume its output, with it. An exhausted successor would break the order. +pub(crate) fn decide(groups: impl Iterator) -> Decision { + let mut front_exhausted = None; + let mut successor_exhausted = false; + for group in groups { + if !group.innocent || group.blamed { + return Decision::Refuse; + } + if !group.abandoned { + return Decision::Wait; + } + match front_exhausted { + None => front_exhausted = Some(group.exhausted), + Some(_) => successor_exhausted |= group.exhausted, + } + } + match front_exhausted { + Some(true) => Decision::Replay, + Some(false) if !successor_exhausted => Decision::Replay, + _ => Decision::Refuse, + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] +mod tests { + use super::*; + fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { + Group { innocent, blamed: false, abandoned, exhausted } + } + const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; + const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + #[test] + fn every_group_abandoned_and_innocent_is_replayed() { + assert_eq!(decide([OK].into_iter()), Decision::Replay); + assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); + } + #[test] + fn nothing_retained_is_never_replayed() { + assert_eq!(decide([].into_iter()), Decision::Refuse); + } + #[test] + fn guilty_work_is_never_replayed() { + assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); + } + #[test] + fn blamed_work_is_never_replayed_even_when_spared() { + // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. + assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); + assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); + // Also once the replay limit is reached, where an innocent front alone is replayed. + let exhausted = Group { exhausted: true, ..BLAMED }; + assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); + } + #[test] + fn a_live_group_waits_and_stops_the_walk() { + assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); + // Nothing after the live group is looked at, guilty or not. + assert_eq!( + decide([group(true, false, false), group(false, true, false)].into_iter()), + Decision::Wait + ); + let mut seen = 0; + let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); + assert_eq!(decide(groups), Decision::Wait); + assert_eq!(seen, 1); + } + #[test] + fn an_exhausted_successor_refuses_an_exhausted_front_replays() { + assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); + assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); + assert_eq!( + decide([group(true, true, true), group(true, true, true)].into_iter()), + Decision::Replay + ); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9f..6f4a5355d6eb39 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,6 +25,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -70,20 +71,20 @@ impl Registry { /// registry on the probe stack. No additional allocation is required. pub(super) fn new( clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, - render: render::RenderState::new(clusters, descriptor_flags)?, + admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -121,7 +122,8 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -129,7 +131,14 @@ impl Registry { Rebind::Fresh }); } + // Physical channels are a device budget, not a permanent entitlement + // of each open logical queue. Only exact idle retirement can yield one. + for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -137,6 +146,13 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -167,6 +183,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -174,6 +191,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -267,6 +288,7 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -275,6 +297,17 @@ impl Registry { Ok(freed) } + /// Builders and released queues waiting for their exact retirement or pool + /// witness temporarily occupy identities. A failed admission must let those + /// transactions finish rather than fail a healthy successor immediately. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -292,11 +325,13 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or(ENOSPC)?; + .ok_or_else(|| self.compute_admission_error( + crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -373,6 +408,32 @@ pub(super) enum Rebind { Fresh, } + +#[kunit_tests(asahi_g17_uninstalled_cleanup)] +mod uninstalled_cleanup_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + #[test] + fn empty_reservation_prevents_reuse_until_explicit_cancellation() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let first = registry.reserve_compute(1).unwrap(); + assert!(registry.reserved(first).is_ok()); + assert!(registry.reserve_compute(1).is_err()); + let second = registry.reserve_compute(2).unwrap(); + assert_ne!(first.id.qid(), second.id.qid()); + registry.cancel_compute(first).unwrap(); + assert!(registry.reserved(first).is_err()); + let next = registry.reserve_compute(3).unwrap(); + assert_eq!(first.id.qid(), next.id.qid()); + assert!(first.id != next.id); + } +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -380,6 +441,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + closed: core::sync::atomic::AtomicBool, } impl Backend { @@ -389,24 +451,54 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -429,6 +521,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -438,13 +531,15 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, admission: &Admission<'_>) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -463,7 +558,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } }; let alloc = super::object::Allocator { @@ -474,7 +571,9 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -507,10 +606,12 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, + admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -522,14 +623,18 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -551,8 +656,23 @@ impl Backend { result } - fn grow_compute(&self, required: usize) -> Result { + /// The caller keeps its preparation lease through this two-phase rollback. + /// An empty registry entry prevents reusing the QID/free-list identity while + /// its old aliases still occupy canonical or client addresses. Recovery joins + /// this lease before removing the session, even if its epoch was invalidated. + fn finish_uninstalled_compute( + &self, retired: Option<(ComputeReservation, KBox)>, + ) -> Result { + let Some((reservation, queue)) = retired else { return Ok(()); }; + drop(queue); + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.queues.cancel_compute(reservation) + } + + fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { use kernel::time::{delay::fsleep, Delta}; + admission.check()?; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -560,45 +680,79 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); - loop { + let mut growth = None; + let result = (|| loop { let result = (|| { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } - if queue.begin_growth(&firmware.init, required)? { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + if growth.is_none() { + let notify = queue.begin_growth(&firmware.init, required)?; + // Capture physical identity before any following fallible + // operation. Close can remove the owner-keyed lookup. + growth = queue.growth_token(); + if notify { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + Err(error) if error == EAGAIN && lease.is_current() => { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } Err(error) => return Err(error), } + })(); + if result.is_err() { + if let Some(token) = growth { + let mut retired = None; + let mut state = self.shared.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + for index in 0..firmware.queues.compute.len() { + let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; + let Some(queue) = entry.queue.as_deref_mut() else { continue; }; + if queue.abort_growth(token)? == Some(true) { + // Keep the empty entry and QID reserved until all + // graph aliases have been destroyed outside the lock. + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + break; + } + } + } + drop(state); + // Tentative replacement mappings also belong to this builder. + // Discard them before a fresh QID can acquire a successor. + drop(alias.take()); + drop(pages.take()); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + } } + result } - fn build_compute(&self, required: usize) -> Result { + fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; + self.admission_open()?; loop { - match self.shared_compute() { + match self.shared_compute(admission) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -607,10 +761,12 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -619,12 +775,42 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context, required)? { + self.admission_open()?; + let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { + Err(error) if error == ENOSPC => { + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; + match rebound { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // The event worker owns release observation and + // drops vacated aliases/backing off this mutex. + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } @@ -638,13 +824,32 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - let ring = compute::DescriptorRing::new(&alloc)?; + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + Ok((pages, compute::DescriptorRing::new(&alloc)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => { + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // A competing ready owner may have taken/rearmed it while + // backing was built off-lock. Leave rejected backing local + // so it is dropped after the device guard, not in rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm()); + if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -660,10 +865,12 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } + self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -671,8 +878,15 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + self.compute_scheduler(&alloc, &lease, admission)?; + let queue = compute::Queue::new( + &alloc, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + self.shared.operand_state, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -682,12 +896,14 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - firmware.queues.install_compute(reservation, &mut built) + admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) } }; // The old aliases must disappear before a waiting constructor can @@ -703,9 +919,11 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result != Err(EAGAIN) { - return result; + if result == Err(ENOSPC) { + self.wait_compute_capacity(admission)?; + continue; } + if result != Err(EAGAIN) { return result; } } } } @@ -780,6 +998,9 @@ impl DeferredBatch { drop(release); } } + pub(super) fn vacated_room(&self) -> bool { + self.vacated.len() < self.vacated.capacity() + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -847,7 +1068,7 @@ struct ComputeHost<'a> { } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_compute_publication() + self.qids.next_publication() } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -918,6 +1139,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, @@ -1053,12 +1283,14 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; for _ in 0..SPARED_WAIT_MS { + admission.check()?; let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1070,7 +1302,10 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + match self.build_compute(required, admission) { + Err(error) if error == EAGAIN => continue, + result => return result, + } } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. @@ -1078,6 +1313,7 @@ impl Backend { } // No new command has been accepted. Keep the spared queue and its // retained work intact until a later admission observes its witness. + admission.check()?; Err(EBUSY) } @@ -1089,23 +1325,48 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; - let mut outer_wait = None; + let mut state_delay = 20i64; + let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let check = || { + if packet.cancel_requested() { return Err(ECANCELED); } + self.admission_open()?; + if packet.completion.status().get() != 0 { return Err(EIO); } + Ok(()) + }; + let admission = Admission::new(deadline, &check); loop { - if packet.completion.status().get() != 0 { - return Err(EIO); + admission.check()?; + // All DRM dependencies were satisfied before this scheduler callback. + // Recheck on every retry: pressure may yield an idle binding between + // admission and publication without changing this logical queue. + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(&admission)?; + // Work states free as this context's earlier commands retire. + // Wait for one under the packet's admission deadline, as render + // publication does, instead of failing the packet. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + admission.check()?; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - if packet.cancel_requested() { - return Err(ECANCELED); - } + admission.check()?; if self .shared .crashed @@ -1126,7 +1387,13 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if firmware.queues.compute(self.owner).is_none() { + drop(state); + drop(lease); + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), @@ -1149,11 +1416,6 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. drop(state); @@ -1199,6 +1461,40 @@ impl super::Firmware { } impl super::Shared { + /// Charge only real watchdog expirations, after the per-job allowance. A + /// changing client or recovery generation cannot mint a fresh storm budget. + pub(super) fn watchdog_timeout(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { return true; } + let stop = { + let mut state = self.state.lock(); + state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) + }; + if stop { self.lose_device_quietly(); } + stop + } + + /// Workload-triggered stop: halt producers first, then join only for a + /// bounded interval. A stalled borrower leaves stopped firmware and all its + /// mappings owned in state. Only a later joined stop may detach/free them. + pub(super) fn stop_runtime(&self) { + let Some(_stopping) = self.stop.try_lock() else { return; }; + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } + firmware.primary.stop(); + } + } + if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { + dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); + return; + } + let firmware = self.state.lock().take(); + if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } + } + /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -1211,7 +1507,9 @@ impl super::Shared { let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() @@ -1259,12 +1557,23 @@ impl super::Firmware { } } self.queues.observe_pool_releases(&self.init, deferred)?; + if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_stalled(now)) { + return Err(ETIMEDOUT); + } if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -1276,3 +1585,163 @@ impl super::Firmware { } } + +impl Registry { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.qids.write_capacity(f)?; + self.admission.write_capacity(f)?; + self.buffers.write_capacity(f)?; + writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; + writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; + self.render.write_capacity(f)?; + writeln!(f, "compute_slots used={} total={} reusable={} building={}", + self.compute.iter().flatten().count(), self.compute.len(), + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), + self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; + writeln!(f, "compute_channels used={} total={}", + self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.write_capacity(f)?; + } + Ok(()) + } +} + +impl Backend { + pub(super) fn wait_capacity(&self, started: u64) -> Result { + let check = || self.admission_open(); + let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); + self.wait_compute_capacity(&admission) + } + + fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + admission.check()?; + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + // A short reclaim transaction obeys the same session/epoch gate as + // publication. Never keep this lease while sleeping for capacity. + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { + return Ok(()); + } + for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } + // Consumption, not publication or an elapsed timeout, frees backing. + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + if !firmware.render_control_backpressured() { + let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; + if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } + } + Ok(()) + })(); + deferred.finish(); + self.shared.queue_events(); + match result { + Err(error) if error == EAGAIN => {} + result => result?, + } + admission.check()?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } +} + +#[kunit_tests(asahi_g17_shared_capacity)] +mod shared_capacity_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + + #[test] + fn all_logical_render_owners_fit_beside_shared_compute() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + // Physical compute channels are device-owned and bounded independently + // of how many logical clients have queued unsignalled work. + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); + // Every accepted logical render owner now has its physical pair; no + // published identity was cancelled or retyped to achieve the fit. + } + + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init( + Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), + GFP_KERNEL, + ) + .unwrap(); + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + + #[test] + fn four_clients_have_capacity_before_any_owner_closes() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); + for client in 0..4u64 { + for queue in 0..9u64 { + let owner = 1 + client * 9 + queue; + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + } + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert_eq!(leases.len(), 36); + } +} + +impl Backend { + /// Logical close is independent of whether an off-lock builder has already + /// installed a physical graph. No constructor can resurrect a closed owner. + fn admission_open(&self) -> Result { + if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } + if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !self.context.is_current() { return Err(EFAULT); } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f1..7d690470cd7b58 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -18,8 +18,39 @@ impl Backend { } fn stop_after_error(&self) { + // Scheduler/fence callbacks never synchronously join builders or RTKit + // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - self.shared.stop_queues(); + } + + fn rollback_unpublished_compute(&self) -> Result { + // No outer preparation lease survives compute_packet's return. Hold a + // fresh epoch through alias destruction so recovery can join this work. + loop { + let lease = self.shared.preparations.enter(false)?; + let mut retired = None; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { drop(state); drop(lease); continue; } + let mut binding = None; + if let Some(index) = firmware.queues.compute.iter().position(|entry| { + entry.as_ref().and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.owner() == Some(self.owner)) + }) { + let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; + let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? + .rollback_unpublished(self.owner, &self.context)?; + binding = replaced; + if cancel { + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + } + } + drop(state); + drop(binding); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + return Ok(()); + } } /// Complete all failure fanout before signalling this packet's unpublished @@ -36,6 +67,11 @@ impl Backend { } deferred.finish(); if !packet.is_published() { + if packet.engine() == Engine::Compute { + if self.rollback_unpublished_compute().is_err() { + self.stop_after_error(); + } + } packet.completion.fail_unpublished(error); } Err(error) @@ -46,6 +82,9 @@ impl Backend { } fn release_queue(&self) -> Result { + // A builder may own only an off-lock reservation when close arrives. + // Record intent before scanning queues; every later install/rearm checks it. + self.closed.store(true, Ordering::Release); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -75,7 +114,11 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - Backend::ensure_compute(self, scs_pages) + // A reverse timeline chain must be submitted before the host signal. + // Validate here, but do not consume physical capacity behind unsignalled + // DRM dependencies. compute_packet admits it when the job can run. + crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + .map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -147,7 +190,17 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); + if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -177,10 +230,47 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + use crate::g17::timeout::{self, Decision}; + // As for compute, a final retirement pass precedes the attribution, and all completion + // callbacks run after releasing the device mutex. Without a collector the attribution + // goes ahead without that pass, as it did before it existed. + let mut deferred = DeferredBatch::worker().ok(); + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(false); + }; + if let Some(deferred) = deferred.as_mut() { + firmware.scan_renders(deferred)?; + } + // Borrowed-graph and queued-retirement deferrals share the renewal bound. + if timeout::transaction_grace(renewals) + && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) + { + return Ok(true); + } + let verdict = firmware.queues.render_timeout_decision(packet, renewals); + let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); + // As compute does: work still blocked behind another VM's older unfinished work + // after the whole grace is innocent, so its failure must stay non-sticky. The pair's + // quarantine extends the class to the pair's other groups. + if decision == Decision::Spare { + packet.completion.classify(crate::g17::recovery::Class::Spared); + } + if let Some(verdict) = verdict { + if let Some(blocker) = verdict.reported_blocker(renewals) { + dev_warn!(self.shared.dev.as_ref(), + "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, verdict.own.vm, verdict.own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); + } + } + Ok(decision == Decision::Defer) + })(); + if let Some(deferred) = deferred.as_mut() { + deferred.finish(); + } + matches!(result, Ok(true)) } fn try_early(&self, packet: &Arc) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34aec..c407d9576e9c47 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -27,19 +27,22 @@ impl Backend { let Some(own) = queue.timeout_head() else { return Ok(Decision::Own); }; - let others = firmware.queues.compute.iter().flatten() + let computes = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let decision = timeout::classify(own, renewals, others); + let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); + let verdict = timeout::attribute(own, renewals, others); + let decision = verdict.decision; if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + if let Some(blocker) = verdict.reported_blocker(renewals) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", - self.owner, own.published, renewals, decision); + "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, own.vm, own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); } Ok(decision) })(); @@ -50,7 +53,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0d4a29e779b4a8..d71f410c76fb48 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -83,7 +83,7 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; + packet.check_replay_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a335..4fd6e567419860 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,14 +24,17 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); @@ -263,6 +266,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -300,7 +304,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); + self.stop_runtime(); return; } let mut deferred = { @@ -324,7 +328,7 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); } } @@ -345,7 +349,7 @@ impl crate::g17::Shared { }; deferred.finish(); if recover { - self.preparations.wait_drained(); + self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -419,7 +423,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } @@ -444,7 +448,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf20455261..59db76b9f109ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -24,6 +24,7 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -46,7 +47,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -55,6 +62,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -65,7 +78,11 @@ pub(super) struct RenderState { } impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + pub(super) fn new( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + ) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -76,6 +93,7 @@ impl RenderState { Ok(Self { config: Config { clusters, + deflake, descriptor_flags, }, entries, @@ -124,6 +142,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -157,6 +183,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -176,6 +203,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -198,7 +226,114 @@ impl RenderState { } impl Registry { - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } @@ -228,7 +363,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -262,12 +399,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -400,6 +533,11 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -419,6 +557,9 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } + fn next_publication(&mut self) -> Result { + self.firmware.queues.qids.next_publication() + } fn publish_outer_pair( &mut self, priority: u8, @@ -553,6 +694,44 @@ impl crate::g17::Firmware { Ok(()) } + /// A final retirement pass before a render watchdog attributes its timeout, as compute runs + /// scan_compute() first: a head that finished but whose completion the event worker has not + /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred + /// timeout or error, stay with the worker, which observes and settles them in its order. + pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()) + && entry.preparing.is_none() + && !entry.growing + && !entry.pending_timeout + && entry.pending_error.is_none() + }) + { + continue; + } + self.retire_render_slot(slot, None, deferred)?; + self.queues.render.collect_retired(slot)?; + self.settle_render_terminal(slot, deferred)?; + } + Ok(()) + } + pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -763,9 +942,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } @@ -816,4 +1005,103 @@ impl Registry { entry.pending_timeout = true; true } + + /// The oldest published group of every render pair, for cross-VM timeout attribution + /// (crate::g17::timeout). + pub(in crate::g17) fn render_timeout_heads( + &self, + ) -> impl Iterator + '_ { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .filter_map(|pair| pair.timeout_head()) + } + + /// A render watchdog blocked behind another VM's older unfinished work gets the same + /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU + /// stalls on one client, the others' renders then outlive the culprit's teardown instead + /// of all timing out together. + pub(in crate::g17) fn render_timeout_decision( + &self, + packet: &Arc, + renewals: u8, + ) -> Option { + let own = self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.contains_packet(packet)) + .and_then(|pair| pair.timeout_head()); + let computes = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.timeout_head()); + // Without a head of its own the timeout is its own. + own.map(|own| { + crate::g17::timeout::attribute( + own, + renewals, + Iterator::chain(self.render_timeout_heads(), computes), + ) + }) + } +} + +impl RenderState { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + self.pools.write_capacity(f)?; + for (slot, entry) in self.entries.iter().enumerate() { + if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { + context.write_capacity(f, "render", slot)?; + } + } + Ok(()) + } +} + +impl Registry { + /// Give idle private backing to another ready client under capacity pressure. + /// Keep graph/QID/context ownership; the next use rearms the same pool after + /// its consumed release, exactly as eviction-driven reclaim does. + pub(super) fn reclaim_idle_render_pools( + &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { break; } + let Some(pool) = entry.pool.as_ref() else { continue; }; + // None means this pool never acquired a firmware registration; + // otherwise require its exact final release to have been consumed. + if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) + || self.admission.owner_active(entry.reservation.owner) { continue; } + // A sibling or replacement may be building off-lock before it has + // installed its pool reference. Its logical owner also excludes us. + let mut idle = true; + for other in render.entries.iter().flatten().filter(|other| { + other.reservation.owner == entry.reservation.owner + || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) + }) { + if other.constructing || other.growing || other.preparing.is_some() + || other.pending_error.is_some() || other.pending_timeout { + idle = false; + break; + } + if let Some(pair) = other.pair.as_ref() { + if !pair.idle_for_context(pair.context())? { idle = false; break; } + } + } + if idle { + if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } + } + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55ca..e3cb174721aaa1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + request.vm_slot == -1 || request.vm_slot as u32 == context.id() }) }) }) @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,7 +271,10 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c80018..4ede20d316c648 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -152,7 +154,7 @@ impl crate::g17::Shared { drop(state); drop(epoch); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); if let Some(mut pair) = pair { pair.stopped(&mut |event| { event.finish(); @@ -163,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index f7f48e853f2252..4713eb5b6b1522 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -239,3 +239,11 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } + +impl Pools { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "render_pool_leases used={} total={} retained={}", + self.leases.iter().flatten().filter(|lease| !lease.released()).count(), + POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7ad..e2034c31765959 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,13 +29,14 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) @@ -44,6 +46,10 @@ impl BuildResources { } } +#[expect( + clippy::large_enum_variant, + reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." +)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -72,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -133,6 +145,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +183,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -238,10 +259,11 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; Pair::new( - &alloc, + alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -252,7 +274,9 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, + resources.config.deflake, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { @@ -277,7 +301,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -316,9 +340,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -360,7 +384,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -397,6 +421,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -429,9 +456,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -463,10 +490,14 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } firmware.queues.teardown.displace( pair.context(), @@ -475,7 +506,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -499,6 +530,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -556,8 +591,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -568,10 +603,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -583,8 +620,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -660,6 +698,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index ce9771778acd4d..499e3858fcca18 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packet; this pass -//! prepares and publishes the same command again under the device mutex. +//! recovery, fragment end never written) and parks the packets in order; this +//! pass prepares and publishes the same commands again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,79 +27,91 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - let Some((packet, owner, qids, closed, busy)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - )) - }) - else { - continue; - }; - if busy { - continue; + // Publish the parked groups front first; a pause keeps the rest parked. + while self.replay_render_front(slot, deferred)? {} + } + Ok(()) + } + + /// Publishes the oldest parked group of `slot`; returns whether it was published. + fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { + let Some((packet, owner, qids, closed, busy, stalled)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + pair.replay_stalled(crate::g17::now_ns()), + )) + }) + else { + return Ok(false); + }; + if stalled { + return Err(ETIMEDOUT); + } + if busy { + return Ok(false); + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); } - if closed || packet.cancel_requested() || !packet.context.is_current() { + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; + return Ok(false); + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + Ok(true) } - if packet.completion.status().get() != 0 { + Err(error) if error == EBUSY || error == EAGAIN => Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, - }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - continue; - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - } - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - } + Ok(false) } } - Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b932161..24c724cb32470b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -33,7 +37,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as _, + kernel::time::msecs_to_jiffies(2000) as isize, ) }; if completed > 0 { @@ -121,6 +125,14 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { + // Unclassified work counts as innocent, and a blamed group must never be replayed. + // Work an earlier recovery spared is collateral here (classify() blames before its + // collateral check) and stays spared, so its failure stays non-sticky; the blame + // itself is recorded apart from the class, and refuses any replay of it. + packet.completion.note_blamed(); + if !packet.completion.spared() { + packet.completion.classify(crate::g17::recovery::Class::Guilty); + } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c5..b77fc13b7ed2fb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,29 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -23,6 +45,128 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + sleep: bool, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + let started = super::super::now_ns(); + let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + loop { + self.admission_open()?; + if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } + match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + // Atomics only: the callback runs under the pool mutex. + if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + Ok(()) + }) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + + /// A pair is borrowed, still settling, or waiting for the release of the + /// scheduler it last ran for to be consumed and the pair parked. The event + /// worker observes both, so kick it on every poll; time out only after the + /// same bound as a capacity wait. + fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + + /// A closing owner may still hold its pair until exact retirement. Wait + /// outside both the device mutex and preparation gate so the event worker + /// and recovery can return capacity. No packet is firmware-visible here. + fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + self.wait_capacity(started) + } + fn render_with_lease( &self, packet: &Arc, @@ -30,6 +174,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -37,22 +182,17 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; + let capacity_started = super::super::now_ns(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -65,7 +205,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -76,6 +220,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { + if !sleep { + return Err(FALLBACK); + } if state_polls == 1000 { return Err(ETIMEDOUT); } @@ -88,19 +235,23 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, sleep) { + Err(error) if error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); @@ -109,6 +260,9 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { + if !sleep { + return Err(FALLBACK); + } if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { return Err(ETIMEDOUT); } @@ -116,11 +270,9 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -163,6 +315,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -296,6 +458,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0c..eb04f803619558 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01c..f8020e8962f849 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b1..53518a76e91a47 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index e76dab07f56db7..d951bd164d6e96 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,11 +6,62 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// A borrowed render graph or queued off-lock retirement is not evidence of +/// progress. Share the finite deferral bound with cross-VM attribution. +pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } + +/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. +/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four +/// extra polls give it ten seconds, still one tenth of the legacy scheduler's +/// fallback. This is an execution allowance, not evidence of GPU progress, and +/// is consumed once per scheduler job, independently of cross-VM/replay grace. +pub(crate) const POLL_MS: u32 = 2000; +const EXECUTION_RENEWALS: u8 = 4; + +/// A background control/replay poll has no scheduler fence to expire it. +/// Its timestamp belongs to the exact transaction and is never renewed by work. +pub(crate) fn background_stalled(since: Option, now: u64) -> bool { + since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) +} + +/// Input comes only from fully validated commands. Render metadata covers the +/// VDM half (vertex/tessellation/geometry) and the fragment half independently. +/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. +pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { + compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build +} + +/// Firmware dependencies can hold an untagged helper behind a long command on +/// the other engine. Classify the complete validated ioctl before enqueueing +/// anything; no state or allowance leaks into a later submission. +pub(crate) fn select_submission(commands: impl Iterator) -> bool { + commands.into_iter().any(|enabled| enabled) +} + +/// Only the explicit AS-build bit is newly accepted; reserved and unrelated +/// flags remain invalid on G17P. Return None before any command is published. +pub(crate) fn as_build(flags: u32) -> Option { + const AS_BUILD: u32 = 1 << 2; + if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } +} + +pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= EXECUTION_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - pub(crate) vm: usize, + /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. + pub(crate) vm: u64, pub(crate) published: u64, pub(crate) finished: bool, + /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains + /// it. No later watchdog of its own will settle it. + pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -20,23 +71,156 @@ pub(crate) enum Decision { Spare, } -pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - if own.finished || !others.into_iter().any(|other| { - other.vm != own.vm && !other.finished && other.published < own.published - }) { - Decision::Own - } else if renewals < MAX_RENEWALS { - Decision::Defer - } else { - Decision::Spare +/// Another VM's older unfinished work that can block `own`: live work before work whose queue +/// already failed, the oldest of each first. +pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { + others + .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) + .min_by_key(|other| (other.failed, other.published)) +} + +/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that +/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind +/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only +/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame +/// published before the culprit, which timed out first, would spare the culprit, and a pair that +/// never drains would spare every later timeout. +pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { + match blocker { + _ if own.finished => Decision::Own, + None => Decision::Own, + Some(_) if renewals < MAX_RENEWALS => Decision::Defer, + Some(blocker) if blocker.failed => Decision::Own, + Some(_) => Decision::Spare, + } +} + +/// A watchdog's decision with the heads it rests on, for its log line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Verdict { + pub(crate) decision: Decision, + pub(crate) own: Head, + pub(crate) blocker: Option, +} + +impl Verdict { + /// The blocker a watchdog line names: at the first deferral and once the grace has run out. + pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { + self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) + } +} + +pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { + let blocker = blocker(own, others); + Verdict { + decision: decide(own, renewals, blocker), + own, + blocker, } } -#[cfg(test)] +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; - fn head(vm: usize, published: u64, finished: bool) -> Head { - Head { vm, published, finished } + #[test] + fn selects_compute_both_render_halves_and_builds() { + assert!(!select(false, [false; 2], false)); + assert!(select(true, [false; 2], false)); + assert!(select(false, [true, false], false)); + assert!(select(false, [false, true], false)); + assert!(select(false, [true; 2], false)); + assert!(select(false, [false; 2], true)); + } + #[test] + fn helpers_share_only_the_current_submission_allowance() { + assert!(select_submission([false, true, false].into_iter())); + assert!(select_submission([true, false].into_iter())); + assert!(select_submission([false, true].into_iter())); + assert!(!select_submission([false, false].into_iter())); + assert!(!select_submission([].into_iter())); + let mut helper = 0; + for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } + assert!(!renew_execution(true, true, &mut helper)); + assert!(!renew_execution(true, false, &mut 0)); + } + #[test] + fn build_hint_rejects_reserved_and_unknown_flags() { + assert_eq!(as_build(0), Some(false)); + assert_eq!(as_build(1 << 2), Some(true)); + for bit in 0..32 { + if bit == 2 { continue; } + assert_eq!(as_build(1 << bit), None); + assert_eq!(as_build((1 << bit) | (1 << 2)), None); + } + assert_eq!(as_build(u32::MAX), None); + } + #[test] + fn borrowed_render_and_queued_retirement_cannot_defer_forever() { + for count in 0..=u8::MAX { + assert_eq!(transaction_grace(count), count < MAX_RENEWALS); + } + } + #[test] + fn background_control_and_replay_polls_have_an_absolute_deadline() { + let since = 42; + assert!(!background_stalled(None, u64::MAX)); + assert!(!background_stalled(Some(since), since - 1)); + assert!(!background_stalled(Some(since), since + 9_999_999_999)); + for _ in 0..1024 { + assert!(background_stalled(Some(since), since + 10_000_000_000)); + } + assert!(background_stalled(Some(since), u64::MAX)); + } + fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + attribute(own, renewals, others).decision + } + fn head(vm: u64, published: u64, finished: bool) -> Head { + Head { vm, published, finished, failed: false } + } + fn failed(vm: u64, published: u64) -> Head { + Head { vm, published, finished: false, failed: true } + } + #[test] + fn ray_query_execution_is_bounded_at_ten_seconds() { + let mut renewals = 0; + for poll in 1..=4 { + assert!(renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, poll); + } + for _ in 0..256 { + assert!(!renew_execution(true, true, &mut renewals)); + } + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); + } + #[test] + fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { + for (published, enabled) in [(false, false), (false, true), (true, false)] { + let mut renewals = 0; + assert!(!renew_execution(published, enabled, &mut renewals)); + assert_eq!(renewals, 0); + } + } + #[test] + fn ray_query_and_cross_vm_credits_are_independent_and_finite() { + let mut ray = 0; + let own = head(5, 101, false); + for _ in 0..4 { + assert!(renew_execution(true, true, &mut ray)); + } + for victim in 0..MAX_RENEWALS { + assert!(!renew_execution(true, true, &mut ray)); + assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); + } + #[test] + fn exhausted_or_invalid_credit_never_wraps() { + for value in 4..=u8::MAX { + let mut renewals = value; + assert!(!renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, value); + } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -52,6 +236,53 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] + fn a_blocker_that_already_failed_defers_then_never_spares() { + // An innocent frame published before the culprit timed out first and failed; its + // quarantined pair keeps its head until the firmware drains it. + let culprit = head(7, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); + } + assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); + assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); + // A pair that never drains does not spare any later timeout either. + let drained_never = [failed(5, 100), failed(5, 102)]; + for own in [head(7, 105, false), head(9, 200, false)] { + assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); + } + } + #[test] + fn a_live_blocker_still_spares_beside_a_failed_one() { + let own = head(9, 103, false); + let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; + assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); + assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); + // Failed or not, a blocker must be older, unfinished and of another VM. + let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; + assert_eq!(blocker(own, late.into_iter()), None); + assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); + // Among failed blockers the oldest is reported. + let both_failed = [failed(8, 102), failed(5, 100)]; + assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); + } + #[test] + fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { + let own = head(9, 103, false); + let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); + let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; + assert_eq!(verdict, expected); + assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); + assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); + let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); + assert_eq!(last.decision, Decision::Own); + assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); + let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); + assert_eq!(spared.decision, Decision::Spare); + assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); + // Nothing blocks: nothing to report. + assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); + } + #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef5..b2a6cf6e8bfe37 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -90,10 +90,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +108,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 3bcf8a9692dcca..72238159d04752 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,8 +21,10 @@ use kernel::{ }, // }, error::Result, + new_mutex, prelude::*, sync::aref::ARef, + sync::Mutex, uapi, // }; @@ -40,6 +42,7 @@ use crate::{ AsahiDrmDriver, // }, file, + memacct, mmu, }; @@ -60,6 +63,11 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, + /// Memory accounting and process attribution of the held backing. + charge: memacct::Charge, + /// Serializes overlapping timestamp writes through every binding of this BO. + #[pin] + pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -74,6 +82,9 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. +/// +/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object +/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -99,6 +110,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } + gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -106,7 +118,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result<&'a kernel::scatterlist::SGTable> { +) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -115,9 +127,7 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(gem) - { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { return Err(EINVAL); } Ok(()) @@ -211,6 +221,7 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -229,20 +240,23 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -fn checked_object_size(size: usize) -> Result { - if size == 0 { return Err(EINVAL); } +pub(crate) fn checked_object_size(size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } - fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { + let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - checked_object_size(size)?, + size, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, + pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -250,6 +264,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -259,12 +274,17 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. +/// +/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the +/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, + pages_no_reclaim: bool, + commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -276,6 +296,7 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, + pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -283,6 +304,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + charge: memacct::Charge::user_current(commit), }, )?; @@ -308,16 +330,22 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), + charge: args.charge, + timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. + /// + /// Their memory is not counted: an import's backing belongs to, and is charged by, its + /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, + charge: memacct::Charge::none(), }) } @@ -372,6 +400,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether CPU mappings use the cacheable WRITEBACK policy. + pub(crate) fn is_writeback(&self) -> bool { + self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 + } + pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } @@ -439,3 +472,22 @@ extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } #[cfg(not(CONFIG_DRM_ASAHI_G15))] pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b14..be65af433a6332 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -268,6 +267,11 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; @@ -285,6 +289,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + *crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -303,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { if self.queue_limits().is_some() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + if self.soft_faults() && self.supports_soft_faults() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags @@ -312,32 +320,39 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. + fn vm_destroyed(&self) {} /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 /// T8122 runtime admits any client. @@ -350,6 +365,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -359,12 +381,12 @@ pub(crate) trait Gpu: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -752,6 +774,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -1427,15 +1450,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0bd..2650131a4a2936 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 00000000000000..7a4d6f288dc3da --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), + // Measured on the 20-core M4 Pro; other core counts are refused at probe. + core_layout: CoreLayout { + cores: Some((2, 20)), + operand_state: fw::compute::OperandState::TWO_BY_TEN, + deflake: fw::render::Deflake::TWO_CLUSTERS, + }, +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f22..75925713bf10bb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,10 +4,11 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -23,10 +24,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,13 +36,30 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + // The layouts the A18 Pro has always run with. + core_layout: crate::g17::CoreLayout { + cores: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -67,6 +85,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } @@ -273,7 +293,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -384,6 +404,9 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; + /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the + /// policy word of the user classes is kept (no macOS trace of class 1 yet). + pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -393,7 +416,9 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + // One physical pair per logical owner leaves room for every execution + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809c..bcb9fd32c7d255 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -254,7 +254,8 @@ impl gpu::Gpu for Backend { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } - fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + crate::timestamp::Buffer::firmware( + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range)?) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ca16e18f10fbf5..a60e38bf9adddb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -18,12 +18,13 @@ impl Destination { let guard = objects.lock(); let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; match &*object { - file::Object::TimestampBuffer(mapping) => { + file::Object::TimestampBuffer(buffer) => { + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { return Err(EINVAL); } - Ok(Some(Self { mapping: mapping.clone(), offset })) + Ok(Some(Self { mapping, offset })) } } } diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 7a5587ef6ec786..5f0ab2b4aea4f5 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,39 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +/// +/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per +/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second +/// UAT with another policy would change it for both. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,12 +108,16 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -167,6 +201,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -185,7 +223,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = ((asid as u64) << 48) + let val = (u64::from(asid) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs new file mode 100644 index 00000000000000..9da09968171e4a --- /dev/null +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU memory attribution, accounting and admission limits. +//! +//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the +//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of +//! the task that faulted it in, but the OOM killer scores a task by the resident set +//! of its mm, and GPU-only pages are mapped into no process. A game holding +//! gigabytes of buffers therefore scores like a tiny process, and so does a +//! compositor; which one an OOM kill hits is left to the other terms of the score. +//! +//! This module closes that gap: +//! +//! - Every user object remembers the mm of the process that created it (an `mmgrab` +//! reference, which keeps only the `mm_struct` itself). While the GPU holds the +//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. +//! They are shmem pages, so this is where `oom_badness()`, the OOM report's +//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages +//! are uncharged when idle reclaim or purge releases the backing and when the +//! object is freed; the mm reference is dropped after the last uncharge, so the +//! counter is back to its own value before `__mmdrop()` checks it. +//! - Each client (DRM file) and the device keep totals of created and held bytes, +//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the +//! `gpu_memory` sysfs file next to `agx_stats`. +//! - A new user object is admitted only while its client stays within a per-client +//! limit and the device within a device-wide limit, both fractions of RAM. +//! A refused allocation fails with `ENOMEM`, which userspace reports as +//! out-of-device-memory, instead of pushing the machine into a global OOM. +//! +//! Imported dma-bufs are charged to their exporter and are not counted here. +//! Kernel objects (firmware structures, per-queue buffers and the global heaps) +//! count towards the device total and its limit, but are attributed to no process. + +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + mm::{ + Mm, + RssCounter, // + }, + page::PAGE_SIZE, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::module_parameters; + +/// Device-wide GPU memory totals, in bytes. +/// +/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). +#[repr(C)] +pub(crate) struct Ledger { + /// Size of every live user object (created, not imported). + pub(crate) user_committed: AtomicU64, + /// Size of the live user objects whose backing the GPU currently holds. + pub(crate) user_resident: AtomicU64, + /// Size of the live kernel objects whose backing the GPU holds. + pub(crate) kernel_resident: AtomicU64, + /// User object creations refused by a limit. + pub(crate) refused: AtomicU64, + /// Device-wide limit on user committed plus kernel resident bytes (0: none). + pub(crate) limit: AtomicU64, + /// Per-client limit on committed bytes (0: none). + pub(crate) client_limit: AtomicU64, +} + +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(Ledger, refused) == 24); + assert!(core::mem::offset_of!(Ledger, client_limit) == 40); + assert!(core::mem::size_of::() == 48); +}; + +/// The ledger of the (single) AGX GPU. +pub(crate) static LEDGER: Ledger = Ledger::new(); + +/// The limits that follow from the amount of RAM and two percentages. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct Limits { + /// Device-wide limit in bytes (0: none). + pub(crate) device: u64, + /// Per-client limit in bytes (0: none). + pub(crate) client: u64, +} + +fn pct_of(total: u64, pct: u32) -> u64 { + match pct { + 0 => 0, + // Above 100% cannot be held anyway; treat it as "no limit". + p if p >= 100 => 0, + p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, + } +} + +/// Computes the limits for `total_ram` bytes of RAM. +/// +/// A percentage of 0 or 100 and above disables that limit. A per-client limit +/// is never larger than the device limit. +pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { + let device = pct_of(total_ram, device_pct); + let mut client = pct_of(total_ram, client_pct); + if device != 0 && (client == 0 || client > device) { + client = device; + } + Limits { device, client } +} + +/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). +/// +/// Returns whether the bytes were added. Concurrent callers cannot overshoot. +fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { + let mut cur = counter.load(Ordering::Relaxed); + loop { + let Some(new) = cur.checked_add(bytes) else { + return false; + }; + if limit != 0 && new.saturating_add(extra) > limit { + return false; + } + match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => return true, + Err(actual) => cur = actual, + } + } +} + +fn sub(counter: &AtomicU64, bytes: u64) { + let prev = counter.fetch_sub(bytes, Ordering::AcqRel); + if prev < bytes { + pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + counter.store(0, Ordering::Relaxed); + } +} + +impl Ledger { + pub(crate) const fn new() -> Self { + Self { + user_committed: AtomicU64::new(0), + user_resident: AtomicU64::new(0), + kernel_resident: AtomicU64::new(0), + refused: AtomicU64::new(0), + limit: AtomicU64::new(0), + client_limit: AtomicU64::new(0), + } + } + + /// Sets the limits from the module parameters and the amount of RAM. + pub(crate) fn configure(&self) { + // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. + let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; + // SAFETY: `si_meminfo()` only fills in the structure it is given. + unsafe { kernel::bindings::si_meminfo(&mut info) }; + let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); + let limits = limits_for( + total, + *module_parameters::mem_limit_pct.value(), + *module_parameters::client_mem_limit_pct.value(), + ); + self.set_limits(limits); + pr_info!( + "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + limits.device >> 20, + limits.client >> 20 + ); + } + + /// Admission capacities exported in the optional GET_PARAMS tail. + pub(crate) fn limits(&self) -> Limits { + Limits { + device: self.limit.load(Ordering::Relaxed), + client: self.client_limit.load(Ordering::Relaxed), + } + } + + pub(crate) fn set_limits(&self, limits: Limits) { + self.limit.store(limits.device, Ordering::Relaxed); + self.client_limit.store(limits.client, Ordering::Relaxed); + } + + /// Admits a new user object of `bytes` for `client`. + /// + /// Reserves the bytes against both limits; the returned [`Commit`] gives + /// them back when it is dropped. + pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { + let client_limit = self.client_limit.load(Ordering::Relaxed); + if !try_add_within(&client.committed, bytes, client_limit, 0) { + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + // Kernel objects share the device limit but are never refused. + let kernel = self.kernel_resident.load(Ordering::Relaxed); + if !try_add_within( + &self.user_committed, + bytes, + self.limit.load(Ordering::Relaxed), + kernel, + ) { + sub(&client.committed, bytes); + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + Ok(Commit { + ledger: self, + client: client.clone(), + bytes, + }) + } +} + +/// Per-client (DRM file) totals, in bytes. +pub(crate) struct Client { + /// Size of the live user objects this client created. + committed: AtomicU64, + /// Size of those whose backing the GPU currently holds. + resident: AtomicU64, +} + +impl Client { + pub(crate) fn new() -> Result> { + Ok(Arc::new( + Self { + committed: AtomicU64::new(0), + resident: AtomicU64::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn committed(&self) -> u64 { + self.committed.load(Ordering::Relaxed) + } + + pub(crate) fn resident(&self) -> u64 { + self.resident.load(Ordering::Relaxed) + } +} + +/// Bytes of a user object reserved against the limits. +pub(crate) struct Commit { + ledger: &'static Ledger, + client: Arc, + bytes: u64, +} + +impl Drop for Commit { + fn drop(&mut self) { + sub(&self.client.committed, self.bytes); + sub(&self.ledger.user_committed, self.bytes); + } +} + +enum Kind { + /// An object the DRM core created (a dma-buf import): charged to its exporter. + None, + /// A driver-owned object. + Kernel(&'static Ledger), + /// A user object, attributed to the process that created it. + User { + commit: Commit, + mm: Option>, + }, +} + +/// The accounting state of one GEM object. +pub(crate) struct Charge { + kind: Kind, + bytes: u64, + /// Whether the bytes are currently counted as held by the GPU. + held: AtomicBool, +} + +// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. +unsafe impl Send for Charge {} +// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. +unsafe impl Sync for Charge {} + +impl Charge { + /// An object that is not counted (a dma-buf import). + pub(crate) fn none() -> Self { + Self { + kind: Kind::None, + bytes: 0, + held: AtomicBool::new(false), + } + } + + /// A kernel object of `bytes`. + pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { + Self { + kind: Kind::Kernel(ledger), + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object admitted by `commit`, attributed to `mm` if there is one. + pub(crate) fn user(commit: Commit, mm: Option>) -> Self { + let bytes = commit.bytes; + let mm = if *module_parameters::oom_attribution.value() != 0 { + mm + } else { + None + }; + Self { + kind: Kind::User { commit, mm }, + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object attributed to the current process. + pub(crate) fn user_current(commit: Commit) -> Self { + let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + Self::user(commit, mm) + } + + fn pages(&self) -> isize { + isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) + } + + fn apply(&self, held: bool) { + match &self.kind { + Kind::None => {} + Kind::Kernel(ledger) => { + if held { + ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&ledger.kernel_resident, self.bytes); + } + } + Kind::User { commit, mm } => { + if held { + commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); + commit + .ledger + .user_resident + .fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&commit.client.resident, self.bytes); + sub(&commit.ledger.user_resident, self.bytes); + } + if let Some(mm) = mm { + let pages = self.pages(); + mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); + } + } + } + } + + /// Records that the GPU now holds (or no longer holds) the object's backing. + /// + /// Idempotent: only a change of state is counted. + pub(crate) fn set_held(&self, held: bool) { + if self.held.swap(held, Ordering::AcqRel) != held { + self.apply(held); + } + } +} + +impl Drop for Charge { + fn drop(&mut self) { + // Uncharge before `kind` (and with it the mm reference) is dropped. + self.set_held(false); + } +} + +#[kunit_tests(asahi_memacct)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + const MIB: u64 = 1 << 20; + + #[test] + fn default_limits_follow_ram() { + // The module parameter defaults. + let l = limits_for(8 * GIB, 67, 50); + assert_eq!(l.device, 8 * GIB * 67 / 100); + assert_eq!(l.client, 4 * GIB); + } + + #[test] + fn disabled_and_clamped_limits() { + assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); + assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); + // A client limit above the device limit is clamped to it. + let l = limits_for(8 * GIB, 25, 50); + assert_eq!(l.client, l.device); + // No client limit with a device limit means the device limit. + let l = limits_for(8 * GIB, 50, 0); + assert_eq!(l.client, 4 * GIB); + // No device limit leaves the client limit alone. + let l = limits_for(8 * GIB, 0, 25); + assert_eq!(l, Limits { device: 0, client: 2 * GIB }); + } + + #[test] + fn percentages_do_not_overflow() { + let l = limits_for(u64::MAX, 99, 98); + assert!(l.device > l.client); + assert!(l.device <= u64::MAX / 100 * 99 + 99); + } + + #[test] + fn try_add_respects_limit_and_extra() { + let c = AtomicU64::new(0); + assert!(try_add_within(&c, 6, 10, 0)); + assert!(!try_add_within(&c, 5, 10, 0)); + assert!(try_add_within(&c, 4, 10, 0)); + assert_eq!(c.load(Ordering::Relaxed), 10); + let d = AtomicU64::new(0); + assert!(!try_add_within(&d, 6, 10, 5)); + assert!(try_add_within(&d, 5, 10, 5)); + let e = AtomicU64::new(u64::MAX - 1); + assert!(!try_add_within(&e, 2, 0, 0)); + assert!(try_add_within(&e, 1, 0, 0)); + } + + static ADMIT: Ledger = Ledger::new(); + + #[test] + fn admission_reserves_and_releases() -> Result { + ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let c = Client::new()?; + let a1 = ADMIT.admit(&a, 40 * MIB)?; + // The client limit refuses a's second 40 MiB; nothing stays reserved. + assert!(ADMIT.admit(&a, 40 * MIB).is_err()); + assert_eq!(a.committed(), 40 * MIB); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); + let b1 = ADMIT.admit(&b, 50 * MIB)?; + // The device limit refuses c's 20 MiB and rolls back c's reservation. + assert!(ADMIT.admit(&c, 20 * MIB).is_err()); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); + assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); + drop(a1); + assert_eq!(a.committed(), 0); + let c1 = ADMIT.admit(&c, 20 * MIB)?; + drop((b1, c1)); + assert_eq!(b.committed(), 0); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + static KERNEL: Ledger = Ledger::new(); + + #[test] + fn kernel_objects_count_against_the_device_limit() -> Result { + KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); + let k = Charge::kernel(&KERNEL, 48 * MIB); + k.set_held(true); + let a = Client::new()?; + assert!(KERNEL.admit(&a, 32 * MIB).is_err()); + let c = KERNEL.admit(&a, 16 * MIB)?; + drop(k); + assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); + drop(c); + Ok(()) + } + + static HELD: Ledger = Ledger::new(); + + #[test] + fn held_transitions_are_idempotent() -> Result { + HELD.set_limits(Limits { device: 0, client: 0 }); + let a = Client::new()?; + let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); + assert!(!charge.held.load(Ordering::Relaxed)); + charge.set_held(true); + charge.set_held(true); + assert_eq!(a.resident(), 8 * MIB); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); + charge.set_held(false); + charge.set_held(false); + assert_eq!(a.resident(), 0); + charge.set_held(true); + // Dropping a held object uncharges it and returns its commit. + drop(charge); + assert_eq!(a.resident(), 0); + assert_eq!(a.committed(), 0); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); + assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + #[test] + fn imports_are_not_counted() { + let charge = Charge::none(); + charge.set_held(true); + assert!(charge.held.load(Ordering::Relaxed)); + drop(charge); + } + + static MM: Ledger = Ledger::new(); + + // `mm_alloc()` is only exported to modules in the KUnit namespace. + #[cfg(not(MODULE))] + #[test] + fn held_pages_are_charged_to_the_creating_mm() -> Result { + MM.set_limits(Limits { device: 0, client: 0 }); + // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. + let raw = unsafe { kernel::bindings::mm_alloc() }; + if raw.is_null() { + return Err(ENOMEM); + } + // SAFETY: `raw` is a live mm until the `mmput()` below. + let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); + let a = Client::new()?; + let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); + assert_eq!(mm.rss_counter(RssCounter::Anon), 0); + charge.set_held(false); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + // Freeing a held object uncharges it before dropping its mm reference. + drop(charge); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + drop(mm); + // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then + // checks that every RSS counter is back to zero. + unsafe { kernel::bindings::mmput(raw) }; + Ok(()) + } +} + +#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] +mod exported_limits_tests { + use super::*; + + #[test] + fn exported_capacities_match_admission() { + let ledger = Ledger::new(); + assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); + let limits = limits_for(8 << 30, 67, 50); + ledger.set_limits(limits); + assert_eq!(ledger.limits(), limits); + assert!(limits.client < limits.device); + assert_eq!(limits.client, 4 << 30); + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bf4a6823024de..cda3ac8d206bd7 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -115,14 +114,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +156,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -363,6 +421,8 @@ struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } @@ -375,7 +435,20 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -494,22 +567,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -556,6 +623,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -641,7 +709,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -684,7 +753,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -719,14 +793,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -736,6 +803,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -790,15 +870,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -826,10 +910,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -837,7 +946,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -895,8 +1011,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -974,7 +1089,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -982,8 +1097,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1024,6 +1140,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1064,10 +1186,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1117,22 +1236,51 @@ pub(crate) struct TimestampMapping { } impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: gem.owned_vmap::<0>()?, + offset, + }) + } + pub(crate) fn write(&self, value: u64) { + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.mapping.owner().timestamp_lock.lock(); + // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + let pointer = unsafe { base.add(self.offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(self.offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.mapping.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1156,12 +1304,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)? + .page_table + .translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; @@ -1174,10 +1324,7 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + TimestampMapping::new(gem, start) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1292,10 +1439,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1385,6 +1532,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1427,8 +1587,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1437,6 +1616,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1453,6 +1634,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1461,14 +1647,44 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1482,11 +1698,15 @@ impl UatShared { } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1503,20 +1723,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1569,9 +1907,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1654,16 +1992,36 @@ impl HandoffFlush { fn prefetch_sgt_segments( dev: &device::Device, gem: &gem::Object, -) -> Result> { +) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { segments, _lease: sgt.into_lease() }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1686,12 +2044,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1717,6 +2075,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1771,7 +2130,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1784,13 +2146,16 @@ impl Vm { /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1899,9 +2264,7 @@ impl Vm { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } @@ -1962,6 +2325,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1976,17 +2370,16 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let size = object_range.range(); let mut vm_exec = unsafe { @@ -1994,14 +2387,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2022,7 +2407,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2040,9 +2425,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2060,17 +2452,16 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(&gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2082,14 +2473,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2124,9 +2507,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2144,17 +2534,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2172,14 +2561,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2222,6 +2603,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2265,22 +2662,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2485,8 +2895,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2499,9 +2919,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2532,6 +2953,18 @@ pub(crate) struct AdoptedRegions { } impl Uat { + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. fn map_physical(range: &Range, min_size: usize) -> Result { @@ -2581,10 +3014,24 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, + ) -> Result { + Self::new_adopted_with_user_upper(dev, geometry, regions, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, ) -> Result { let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; @@ -2596,8 +3043,12 @@ impl Uat { let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2606,10 +3057,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2621,8 +3076,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2725,7 +3186,10 @@ impl Uat { } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2754,7 +3218,9 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); @@ -2773,16 +3239,24 @@ impl Uat { let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2791,8 +3265,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2811,6 +3289,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for index in UAT_USER_CTX_START..UAT_NUM_CTX { + if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2862,6 +3369,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2871,14 +3379,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2900,7 +3411,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -2920,7 +3431,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -2936,15 +3447,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2961,24 +3476,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -2988,7 +3512,12 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -2997,12 +3526,16 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3011,10 +3544,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3030,12 +3567,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3043,11 +3595,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3098,11 +3650,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3114,7 +3670,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3123,8 +3679,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3142,3 +3702,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = lock_vm(&self.gpuvm, None, false)?; + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 44a1575ecfd2b7..e222c7cd8e6a98 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,6 +17,7 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, + _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -57,7 +58,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state without taking DMA reservations or changing any leaf. + /// Prepares host state before taking the GPUVM exec lock or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -80,24 +81,20 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - crate::gem::held_sg_table(gem, dev)?; + let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - { - let mut segments = vm_bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetch_sgt_segments(dev, gem)?); - } - } + prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, + _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 30010be497973d..66d507c2d76949 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,9 +79,13 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 && state.commits == 0 - && !state.blocked && !state.quarantined && !state.draining - && !state.closed && !state.cleanup_failed + state.active == 0 + && state.commits == 0 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed }) } @@ -339,10 +343,12 @@ impl Vm { drop(previous); } - /// Whether userspace has permanently closed this VM's map admission. - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed + /// admission for good). G15 releases its per-VM state after that. pub(crate) fn is_closed(&self) -> bool { - self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -411,7 +417,10 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); + return Ok(VmJobGuard { + vm: self.clone(), + _residency: residency, + }); } lifetime.changed.wait(&mut state); } @@ -471,7 +480,11 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { + pub(super) fn defer_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + start: usize, + ) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -485,7 +498,8 @@ impl Vm { } if batch .ops - .iter().skip(start) + .iter() + .skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -496,16 +510,21 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = + self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + if !state + .unmaps + .iter() + .any(|old| old.iova == unmap.iova && old.size == unmap.size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -530,16 +549,30 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { return Err(ENOENT); } - if !state.blocked { return Ok(false); } - if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -634,8 +667,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); - let objects = core::mem::replace(&mut state.objects, KVec::new()); + let unmaps = core::mem::take(&mut state.unmaps); + let objects = core::mem::take(&mut state.objects); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -671,7 +704,9 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { status.record(error); } + if let Some(status) = &self.status { + status.record(error); + } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a8134268..8611221af492a3 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs new file mode 100644 index 00000000000000..7cdad5e1f3831d --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/ppl.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire +//! guard lifetime: firmware exclusion alone does not serialize two AP threads. +//! This core has no kernel dependencies so the production guard can be exercised +//! against deterministic firmware and concurrent host transports. + +pub(super) trait Memory { + fn firmware_interested(&self) -> bool; + fn firmware_turn(&self) -> bool; + fn ap_interest(&self, interested: bool); + fn give_firmware_turn(&self); + fn barrier(&self); + fn expired(&self) -> bool; + fn relax(&self); + /// Latch failure before the transport releases host exclusion on timeout. + fn fail(&self); +} + +/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. +#[must_use = "retain the guard until every protected table mutation has completed"] +pub(super) struct Guard { + memory: T, +} + +impl Guard { + pub(super) fn acquire(memory: T) -> Result { + let timeout = || { + memory.ap_interest(false); + memory.barrier(); + memory.fail(); + Err(()) + }; + memory.ap_interest(true); + memory.barrier(); + while memory.firmware_interested() { + if memory.expired() { + return timeout(); + } + if memory.firmware_turn() { + memory.ap_interest(false); + memory.barrier(); + while memory.firmware_turn() { + if memory.expired() { + return timeout(); + } + memory.relax(); + } + memory.ap_interest(true); + memory.barrier(); + } + memory.relax(); + } + memory.barrier(); + Ok(Self { memory }) + } + + pub(super) fn memory_mut(&mut self) -> &mut T { + &mut self.memory + } +} + +impl Drop for Guard { + fn drop(&mut self) { + self.memory.give_firmware_turn(); + self.memory.barrier(); + self.memory.ap_interest(false); + self.memory.barrier(); + // Only after these stores complete may `memory` release the host mutex. + } +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 5370b87ccf1b5d..49c2241bc399fa 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,6 +251,8 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} +// SAFETY: Shared callbacks use the VM's residency/reservation locks. The +// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -273,7 +275,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -281,23 +283,36 @@ impl VmShrinker { Ok(owner) } + /// Count reclaimable pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker. The core must + /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() as _ + owner.vm.idle_reclaim_count() } + /// Reclaim idle pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker, and `sc` must + /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - let target = unsafe { (*sc).nr_to_scan } as usize; - owner.vm.reclaim_idle_pages(target) as _ + // SAFETY: The shrinker core supplies a live initialized control. + let target = unsafe { (*sc).nr_to_scan }; + owner.vm.reclaim_idle_pages(target) } } @@ -348,7 +363,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -361,11 +376,17 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Purgeable objects count on busy VMs too; idle backing only on idle ones. - if object.idle_reclaim_candidate() - && (object.madv_locked() > 0 || idle) - && object.sole_sg_lease_releasable_locked() - { + // Count retained backing on idle VMs, and purgeable contents even if idle + // reclaim already released the table. Never wait on the BO mutex under resv. + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + let advised = object.madv_locked() > 0; + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + (advised || object.has_sg_table_locked()) + && object.unleased_pages_releasable_locked() + }; + if object.idle_reclaim_candidate() && (advised || idle) && releasable { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -385,7 +406,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -396,7 +417,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -408,9 +429,12 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_some() && !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -467,7 +491,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) || !self .lifetime .as_ref() @@ -482,7 +506,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -494,9 +518,14 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_none() || !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + // A competing lease may have made the previous release fail after we + // dropped our segments. Retry that table, but do not count empty backing. + object.has_sg_table_locked() && object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index b9e2975c898ff4..fec7a6b61ad8fb 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: gem as *const gem::Object as usize, + object: core::ptr::from_ref(gem) as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = gem as *const gem::Object as usize; + let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 38a2268137effb..a66a82d92a5830 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -409,7 +409,7 @@ impl> GpuObject { }; let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index b88353a2f73441..c5a47c24ec9286 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -53,19 +54,20 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). +/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local +/// CTR_EL0 read can select too large a stride if the task migrates during the loop. #[inline(always)] fn dcache_line_size() -> usize { - let ctr: u64; - // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. - unsafe { - core::arch::asm!( - "mrs {ctr}, ctr_el0", - ctr = out(reg) ctr, - options(nomem, nostack, preserves_flags) - ) - }; - 4 << ((ctr >> 16) & 0xf) + use kernel::bindings; + let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) + | (bindings::SYS_CTR_EL0_Op1 << 16) + | (bindings::SYS_CTR_EL0_CRn << 12) + | (bindings::SYS_CTR_EL0_CRm << 8) + | (bindings::SYS_CTR_EL0_Op2 << 5); + // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine + // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. + let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; + 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -95,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -116,20 +117,114 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). +/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -137,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -253,6 +351,8 @@ impl Prot { Self { memattr, ..*self } } + /// The protection bits of a leaf with this protection (no address, table type or non-global + /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() @@ -278,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -296,6 +412,12 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Ownership could not be relinquished to firmware; physical tables remain until reboot. + retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -305,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -317,45 +438,49 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + retained: false, + reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb osh`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// `dsb sy`. Call it before anything is mapped and before any context names the root. pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -366,6 +491,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -381,7 +507,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -392,15 +520,23 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + retained: false, + reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } + /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim + /// this tree merely because a userspace handle closed or a fence reported an error. + pub(crate) fn retain(&mut self) { + self.retained = true; + } + pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -470,6 +606,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -513,55 +650,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); } - })?; + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -597,13 +737,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -611,7 +748,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -636,16 +778,9 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { + fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } + tables.contains(&phys) } else { true }; @@ -653,32 +788,44 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; + let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; tables.swap_remove(index); } + if self.cleans_poc() { + // Complete the parent descriptor clear before its detached child can be reused. + // Callers have already removed this tree from live contexts before freeing tables. + poc_barrier(); + } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -769,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -788,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -814,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -861,6 +1019,9 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { + if self.retained { + return; + } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index a68352828cfbc3..20396f2e772252 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.clone(), + mapping: mapping.firmware_mapping()?, offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 0872a50ce5d463..4abdd3e485f726 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -262,7 +262,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { QueueTimeout { dev: self.dev.clone(), id: self.id } @@ -754,7 +755,22 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - cmdbuf.skip(header.size as usize); + let payload_size = header.size as usize; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; + if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; + if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + _ => cmdbuf.skip(payload_size), + } nr_commands += 1; match header.cmd_type as u32 { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index fb22569ea5fdff..0a4ee8e7563ec2 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,10 +13,11 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, c_str, + bindings, + c_str, device::Core, - devres::Devres, error::{from_err_ptr, to_result}, + devres::Devres, io::{ mem::IoMem, Io, @@ -222,7 +223,10 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + from_err_ptr(bindings::apple_mbox_get( + pdev.as_ref().as_raw(), + index.try_into()?, + ))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -235,7 +239,10 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { dev: pdev.into(), mbox }) + Ok(Self { + dev: pdev.into(), + mbox, + }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -251,7 +258,11 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + dev_err!( + self.dev.as_ref(), + "Could not stop GPU coprocessor: {:?}\n", + error + ); } } } @@ -560,7 +571,10 @@ impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + let sgx = Arc::new( + sgx_req.iomap_sized::()?.into_devres()?, + GFP_KERNEL, + )?; Ok(G17Resources { dev: pdev.into(), @@ -701,7 +715,6 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } - } /// Split the packed per-cluster core masks into one mask per cluster. @@ -732,11 +745,7 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); + dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); return Err(EIO); } @@ -744,7 +753,10 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision( + dev: &platform::Device, + id_version: u32, +) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -758,3 +770,29 @@ fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevis } }) } + +/// Device-managed SGX mapping retained by the G16 firmware session. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[derive(Clone)] +pub(crate) struct G16Resources { + sgx: Arc>>, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl G16Resources { + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; + Ok(Self { + sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, + }) + } + pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_read32(offset) + } + #[allow(dead_code)] + pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_write32(value, offset) + } +} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 9f0baa4350d93f..7d7d785a831334 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -32,9 +32,21 @@ struct asahi_stats_snapshot { u64 jobs; }; +/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ +struct asahi_gpu_memory { + u64 user_committed; + u64 user_resident; + u64 kernel_resident; + u64 refused; + u64 limit; + u64 client_limit; +}; + struct asahi_stats_attribute { struct device_attribute attr; + struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; + const struct asahi_gpu_memory *memory; bool export_enabled; }; @@ -72,20 +84,52 @@ static ssize_t agx_stats_show(struct device *dev, return n; } +/* + * GPU memory held by the driver, in bytes: user objects created (committed) + * and held by the GPU (resident), held kernel objects, the limits that admit + * user objects (0: none) and the number of refused creations. + */ +static ssize_t gpu_memory_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, memory_attr); + const struct asahi_gpu_memory *mem = stats->memory; + ssize_t n = 0; + + n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", + (unsigned long long)READ_ONCE(mem->user_committed)); + n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", + (unsigned long long)READ_ONCE(mem->user_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", + (unsigned long long)READ_ONCE(mem->kernel_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", + (unsigned long long)READ_ONCE(mem->limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", + (unsigned long long)READ_ONCE(mem->client_limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", + (unsigned long long)READ_ONCE(mem->refused)); + + return n; +} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle) + int export_enabled, const void *memory, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || (export_enabled && !snapshot)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); + BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -102,17 +146,70 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->snapshot = snapshot; stats->export_enabled = export_enabled; + sysfs_attr_init(&stats->memory_attr.attr); + stats->memory_attr.attr.name = "gpu_memory"; + stats->memory_attr.attr.mode = 0444; + stats->memory_attr.show = gpu_memory_show; + stats->memory = memory; + ret = device_create_file(dev, &stats->attr); if (ret) { kfree(stats); return ret; } + ret = device_create_file(dev, &stats->memory_attr); + if (ret) { + device_remove_file(dev, &stats->attr); + kfree(stats); + return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); +/* + * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores + * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores + * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs + * about 1 ms, so userspace may lengthen it on AC power. + */ +static u32 *asahi_idle_word; + +static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); +} + +static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + u32 ms; + + if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) + return -EINVAL; + WRITE_ONCE(*asahi_idle_word, ms); + return count; +} + +static DEVICE_ATTR_RW(idle_off_delay_ms); + +int asahi_sysfs_register_idle(struct device *dev, u32 *word) +{ + int ret; + + if (!dev || !word) + return -ENODEV; + asahi_idle_word = word; + ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); + if (ret) + asahi_idle_word = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); + void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -121,7 +218,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) return; /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); + if (asahi_idle_word) { + device_remove_file(dev, &dev_attr_idle_off_delay_ms); + asahi_idle_word = NULL; + } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 5969a1718925f1..f1873f9be6caaa 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -6,7 +6,8 @@ struct device; int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle); + int export_enabled, const void *memory, void **handle); +int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 079623d1ef2108..cee5d52a4e2619 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,7 +3,7 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::stats::StatsSnapshot; +use crate::{memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; extern "C" { @@ -11,12 +11,30 @@ extern "C" { dev: *mut c_void, snapshot: *const StatsSnapshot, export_enabled: c_int, + memory: *const Ledger, handle: *mut *mut c_void, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } +extern "C" { + fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; +} + +/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). +pub(crate) fn register_idle( + dev: *mut kernel::bindings::device, + word: *mut u32, +) -> kernel::error::Result { + // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes + // it with READ_ONCE/WRITE_ONCE. + let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; + if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } +} + /// Register only after initialization succeeds, keeping the snapshot Arc alive. +/// +/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, @@ -30,6 +48,7 @@ pub(crate) fn register( dev.cast(), snapshot, c_int::from(export_enabled), + &crate::memacct::LEDGER, &mut handle, ) }; diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs new file mode 100644 index 00000000000000..976ec9f6bf48be --- /dev/null +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared timestamp destinations shared by native command completion paths. + +use crate::mmu; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use crate::{file, gem}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::ops::Range; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::sync::atomic::{fence as barrier, Ordering}; +use kernel::{prelude::*, sync::Arc}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use kernel::{uapi, xarray}; + +/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. +/// CPU-retired timestamps retain their backing without consuming global firmware address space. +pub(crate) enum Buffer { + Firmware(Arc), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Cpu { + // The GEM pin keeps the object's held SG table alive through CPU publication. + gem: gem::KernelMappingPin, + range: Range, + }, +} + +impl Buffer { + pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { + Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + } + + /// Retain a CPU-written byte range without publishing any firmware address. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { + let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { + return Err(EINVAL); + } + let gem = gem::KernelMappingPin::new(&bo.gem); + let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let _ = gem::held_sg_table(&gem, parent.as_ref())?; + Ok(Self::Cpu { gem, range }) + } + + pub(crate) fn size(&self) -> usize { + match self { + Self::Firmware(mapping) => mapping.size(), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { range, .. } => range.end - range.start, + } + } + + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + match self { + Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { gem, range, .. } => { + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) + } + } + } + + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. + pub(crate) fn firmware_mapping(&self) -> Result> { + match self { + Self::Firmware(mapping) => Ok(mapping.clone()), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { + let offset = usize::try_from(offset).map_err(|_| EINVAL)?; + let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; + if aligned && offset % size_of::() != 0 { + return Err(EINVAL); + } + if end > size { + return Err(if aligned { EINVAL } else { ERANGE }); + } + Ok(offset) +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +struct Destination { + // Retain the bound range and backing along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + aligned: bool, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { + _mapping: mapping, + cpu, + })) + } + + fn write(&self, ticks: u64, frequency_hz: u64) { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / frequency_hz) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); + self.cpu.write(ns); + barrier(Ordering::Release); + } +} + +/// Timestamp mappings resolved and retained before any command is enqueued. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct Destinations(Option; 4]>>); + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destinations { + /// G16 retains its eight-byte destination alignment and EINVAL contract. + pub(crate) fn resolve_aligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, true) + } + + fn resolve_inner( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + aligned: bool, + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + /// Publish measured timestamps before signalling completion or releasing dependent work. + pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { + for (destination, ticks) in self.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks, frequency_hz); + } + } + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = self + .0 + .as_deref() + .map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573cc..a5f9455b23f5fd 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -217,3 +217,17 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } + +/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +#[inline(always)] +pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__pinned_init(slot) } +} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e50..9de2e7f0f4f97c 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,8 +87,16 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + /* A bulk pin must not run direct reclaim once per folio when + * the driver can instead return an allocation failure. Clearing + * DIRECT_RECLAIM also makes memcg charges fail without reclaim + * or invoking the OOM killer. Preserve placement and kswapd. + */ + if (shmem->pages_no_reclaim) + gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); + mapping_set_gfp_mask(obj->filp->f_mapping, gfp); } return 0; @@ -612,10 +620,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + return !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == 1 && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1075,6 +1083,10 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; + /* Purged contents cannot be restored by rebuilding an empty table. */ + if (shmem->madv < 0) + return ERR_PTR(-EINVAL); + drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa68..30791d328550de 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,6 +37,13 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; + /** + * @pages_no_reclaim: Disable direct reclaim during backing population. + * Set before drm_gem_shmem_init(). Drivers that can fail a large pin + * request cheaply may opt in; background reclaim remains enabled. + */ + bool pages_no_reclaim; + /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -145,7 +152,9 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index d311b92620b864..66396f89cc3339 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,6 +212,20 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; + + /** + * @memory_limit: Device-wide limit on user committed plus kernel + * resident bytes. Zero means no driver limit. This is a capacity, + * not current free memory; other clients share it. + */ + __u64 memory_limit; + + /** + * @client_memory_limit: Per-DRM-file limit on user committed bytes. + * Zero means no driver limit. Userspace must zero-initialize the reply + * buffer: older kernels leave these optional tail fields untouched. + */ + __u64 client_memory_limit; }; /** @@ -348,6 +362,12 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, + + /** + * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept + * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. + */ + DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -806,7 +826,8 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Reserved for future extension. + * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails + * with -EACCES. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -1614,6 +1635,17 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + + /** + * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, + * including compute-based transfers. Requires BOUNDED_AS_BUILD. + * This is an execution hint, not proof of progress or permission to + * access additional memory. The kernel validates the command normally + * and caps its execution allowance at ten seconds (four extra 2s polls). + * Other commands in the same submission receive the same allowance so + * ordered helpers cannot time out while waiting for the build. + */ + DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 099e64b3ce5fdb..9ab7494251e289 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index b5540997bd20d4..d9933c4595682a 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,6 +23,18 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } +__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, + long value) +{ + add_mm_counter(mm, member, value); +} + +__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, + int member) +{ + return get_mm_counter_sum(mm, member); +} + __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f2450492..e63b62b54a17d4 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: None, + show_fdinfo: Some(drm::File::::show_fdinfo_callback), fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8a56f00203f568..88709a7f9b36dd 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,6 +12,23 @@ use crate::{ }; use core::marker::PhantomData; +/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. +/// +/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct MemoryStats { + /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). + pub shared: u64, + /// Memory private to the client (part of `drm-total-*`). + pub private: u64, + /// Memory currently resident (`drm-resident-*`). + pub resident: Option, + /// Resident memory userspace marked purgeable (`drm-purgeable-*`). + pub purgeable: Option, + /// Memory in use by active submissions (`drm-active-*`). + pub active: Option, +} + /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -22,6 +39,11 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; + + /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. + fn memory_stats(&self) -> Option { + None + } } /// An open DRM File. @@ -48,6 +70,12 @@ impl File { self.0.get() } + /// Whether this file holds the device's DRM master (the compositor or display server). + pub fn is_current_master(&self) -> bool { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. + unsafe { bindings::drm_is_current_master(self.as_raw()) } + } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -99,6 +127,47 @@ impl File { 0 } + /// The `show_fdinfo` callback of a `struct drm_driver`. + pub(crate) extern "C" fn show_fdinfo_callback( + printer: *mut bindings::drm_printer, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose + // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the + // file, so `postclose_callback()` cannot run concurrently. + let file = unsafe { File::::from_raw(raw_file) }; + let Some(stats) = file.inner().memory_stats() else { + return; + }; + + let mut supported = 0; + let mut raw = bindings::drm_memory_stats { + shared: stats.shared, + private: stats.private, + resident: 0, + purgeable: 0, + active: 0, + }; + if let Some(resident) = stats.resident { + raw.resident = resident; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; + } + if let Some(purgeable) = stats.purgeable { + raw.purgeable = purgeable; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; + } + if let Some(active) = stats.active { + raw.active = active; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; + } + + // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` + // and the region name outlive the call. + unsafe { + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + }; + } + /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483e..83eca852139beb 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,6 +82,12 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, + /// Fail backing population when it would need direct reclaim. + /// + /// Background reclaim and ordinary allocation placement are preserved. + /// Intended for large, fallible pin requests, not essential kernel backing. + pub pages_no_reclaim: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -94,13 +100,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, + pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -116,12 +122,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// Devres object for unmapping any SGTable on driver-unbind. - sgt_res: ManuallyDrop>>>, + /// One cleanup registration for the C SG-table cache; never holds a table borrow. + sgt_cleanup: ManuallyDrop>>>, #[pin] - /// Lock for protecting initialization of `sgt_res`. + /// Serializes cleanup registration and table acquisition. sgt_lock: Mutex<()>, - /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, #[pin] inner: T, @@ -189,13 +195,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_res` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + // `sgt_cleanup` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -273,67 +279,59 @@ impl Object { self.make_vmap() } - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA - /// pages for this object. + /// Acquire a revocable lease on this object's DMA scatter-gather table. /// - /// This will pin the object in memory. It is expected that `dev` should be a pointer to the - /// same [`device::Device`] which `self` belongs to, otherwise this function will return - /// `Err(EINVAL)`. - pub fn sg_table<'a>( - &'a self, - dev: &'a device::Device, - ) -> Result<&'a scatterlist::SGTable> { + /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. + /// The table borrow cannot outlive the lease. The device must be this object's parent. + /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. + pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // SAFETY: `import_attach` is set before an imported object is published and does not - // change until the object is freed. - if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // The SG table of an imported object is the dma-buf attachment's mapping, which the - // exporter made for this device at import time. It belongs to the attachment, and - // `drm_gem_shmem_release()` hands it back to the exporter through - // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` - // would unmap and free it as if these helpers had built it. - // - // SAFETY: `sgt` of an imported object is set before the object is published and stays - // valid and unmodified until the object is freed, which `'a` cannot outlive. + // Release and purge set the reclaim bit only while they hold this object's reservation. + // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. + while let Err(e) = self.acquire_sgt_access() { + if e != EBUSY { + return Err(e); + } + drop(DmaResvGuard::new(self)); + } + // Every failure after incrementing the count drops this lease. + let lease = SGLease { obj: self.into() }; + + // SAFETY: Import attachment ownership is immutable after publication. + let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // Imports belong to the exporter. Never register them for Rust devres cleanup, + // rebuild their tables, or release their backing. + // SAFETY: The attachment table stays valid until this object is freed. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - // SAFETY: See above. - return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); - } - - // The borrowed reference can outlive any lock, so pin the backing permanently. - self.acquire_sgt_access(true)?; - - let sgt_res = 'out: { - // Fast path: sgt_res is already initialized - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; - } - - // Slow path: Grab the lock and see if we need to initialize sgt_res. + sgt + } else { + // Register cleanup once, but cache the actual table only in C's shmem->sgt. + // Reclaim clears that pointer. No Rust reference to it survives without a lease. let _guard = self.sgt_lock.lock(); - - // If someone initialized it while we were waiting, we can exit early. - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; + if self.sgt_cleanup.as_ref().is_none() { + self.sgt_cleanup.populate(Devres::new( + dev, + Ok::<_, Error>(SGTableCleanup { obj: self.into() }), + )?); } - - // If not, finish initializing and return. `populate()` cannot return false, as - // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. - self.sgt_res - .populate(Devres::new(dev, SGTableMap::new(self))?); - - // SAFETY: We just populated sgt_res above. - unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; + // SAFETY: The live object and lease exclude backing release. The helper locks + // the reservation and either returns the cached table or rebuilds it. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? }; - Ok(sgt_res.access(dev)?) + Ok(SGTable { + // SAFETY: The lease excludes release; the Bound device excludes devres revocation. + table: unsafe { scatterlist::SGTable::from_raw(sgt) }, + lease, + }) } /// Create a new shmem-backed DRM object of the given size. @@ -349,7 +347,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -360,6 +358,10 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + // SAFETY: The object is initialized but not published. The C initializer + // reads this policy when setting up the backing mapping's GFP mask. + unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; + // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -403,7 +405,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -429,23 +431,18 @@ impl Object { } } - fn acquire_sgt_access(&self, permanent: bool) -> Result { + fn acquire_sgt_access(&self) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - let next = if permanent { - state | SGT_PERMANENT - } else { - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } - state + 1 - }; + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } match self.sgt_access.compare_exchange_weak( state, - next, + state + 1, Ordering::AcqRel, Ordering::Acquire, ) { @@ -466,11 +463,34 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } + /// Whether the object still has a cached device SG table. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn has_sg_table_locked(&self) -> bool { + // SAFETY: Reservation ownership serializes the C table pointer. + !unsafe { (*self.as_raw_shmem()).sgt }.is_null() + } + + /// Whether backing without a Rust SG lease can be reclaimed or purged. + /// + /// Also accepts backing whose SG table was already released by idle reclaim. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 0 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that - /// borrow may outlive any lock. + /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on + /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. /// /// # Safety /// @@ -516,7 +536,9 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } + || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + { return Err(EBUSY); } self.sgt_access @@ -706,63 +728,72 @@ where { } -/// A reference to a GEM object that is known to have a mapped [`SGTable`]. -/// -/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables -/// on GEM shmem objects are revoked on driver-unbind. -/// -/// # Invariants +/// An owned backing lease, suitable for retaining copied DMA segments. /// -/// - `self.obj` always points to a valid GEM object. -/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an -/// [`SGTable`]. -/// -/// [`SGTable`]: scatterlist::SGTable -pub struct SGTableMap { - obj: NonNull>, +/// Dropping the last lease permits reclaim after device translations have been invalidated. +/// This does not itself provide access to the table: table access also requires a Bound device. +#[must_use = "retain until all device uses of the backing have ended"] +pub struct SGLease { + obj: ARef>, } -impl Deref for SGTableMap { +impl Drop for SGLease { + fn drop(&mut self) { + let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); + debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); + } +} + +/// A leased SG-table borrow. References obtained from it cannot outlive the lease. +#[must_use = "retain while using the SG table or transfer into an owned lease"] +pub struct SGTable<'a, T: DriverObject> { + table: &'a scatterlist::SGTable, + lease: SGLease, +} + +impl Deref for SGTable<'_, T> { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - // SAFETY: - // - The NonNull is guaranteed to be valid via our type invariants. - // - The sgt field is guaranteed to be initialized and valid via our type invariants. - unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + self.table } } -impl Drop for SGTableMap { - fn drop(&mut self) { - // SAFETY: `obj` is always valid via our type invariants - let obj = unsafe { self.obj.as_ref() }; - let _lock = DmaResvGuard::new(obj); - - // SAFETY: We acquired the lock needed for calling this function above - unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; +impl SGTable<'_, T> { + /// Retain the backing after copying the DMA addresses, without retaining a table borrow. + pub fn into_lease(self) -> SGLease { + self.lease } } -impl SGTableMap { - fn new(obj: &Object) -> impl Init { - // INVARIANT: - // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, - // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. - // SAFETY: - // - `obj` is fully initialized, making this function safe to call. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; +/// Device-unbind cleanup registration for locally allocated SG tables. +/// +/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may +/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. +struct SGTableCleanup { + obj: NonNull>, +} - Ok(Self { obj: obj.into() }) +impl Drop for SGTableCleanup { + fn drop(&mut self) { + // SAFETY: The registration is dropped before its GEM object. + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. + // Reclaim may already have freed the table; the C free helper requires a non-null one. + unsafe { + if !(*obj.as_raw_shmem()).sgt.is_null() { + bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); + } + } } } -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Send for SGTableMap {} -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Sync for SGTableMap {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Send for SGTableCleanup {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Sync for SGTableCleanup {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -839,6 +870,14 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: This new faux device is private to the fixture. Use direct DMA with a + // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -885,19 +924,225 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.read8(offset), expected); + assert_eq!(vmap.try_read8(offset).unwrap(), expected); } Ok(()) } - // TODO: I would love to actually test the success paths of sg_table(), but that would require - // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave - // that for someone else. + #[test] + fn sg_lease_counting() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let second = obj.sg_table(dev.as_ref().as_ref())?; + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(second); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn release_idle_without_mappings() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes these fields. + let empty = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + ) + }; + assert!(empty.0); + assert!(empty.1); + // SAFETY: Still no device mappings or leases; an already idle release succeeds too. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn release_refused_while_leased() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no actual device mappings were installed. + assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; no actual device mappings were installed. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(bind); + // SAFETY: Reservation held; the mapping lease is now the sole owner. + assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; the remaining lease must still exclude release. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; no device mapping or SG lease remains. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn purge_madvised_after_last_lease() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no real device access. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; all device uses and SG leases have ended. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the backing fields and advice. + let (empty_sgt, empty_pages, madv) = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + obj.madv_locked(), + ) + }; + assert!(empty_sgt); + assert!(empty_pages); + assert_eq!(madv, -1); + } + assert!(!obj.madvise(0)); + assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn sg_rebuild_after_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.vmap::<4>()?.as_view().write32(0x12345678, 0); + for _ in 0..3 { + let table = obj.sg_table(dev.as_ref().as_ref())?; + assert!(table.iter().count() > 0); + drop(table); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; all leases have dropped, with no device mappings. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes the table pointer. + assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); + } + assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); + } + // Drop with a rebuilt table exercises the surviving devres cleanup registration too. + drop(obj.sg_table(dev.as_ref().as_ref())?); + Ok(()) + } + + #[test] + fn purge_after_idle_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + unsafe { obj.release_idle_pages_locked()? }; + } + assert!(obj.madvise(1)); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + assert!(unsafe { obj.unleased_pages_releasable_locked() }); + // SAFETY: Reservation held; no device access or SG lease remains. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the advice and table pointer. + let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; + assert_eq!(madv, -1); + assert!(empty); + Ok(()) + } + + #[test] + fn purge_refuses_kernel_vmap() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let vmap = obj.vmap::<4>()?; + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; the vmap must prevent the backing from being freed. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + // SAFETY: Reservation held; the vmap must exclude idle release too. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + } + drop(vmap); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device, vmap or SG users remain. + unsafe { obj.purge_locked()? }; + Ok(()) + } + + #[test] + fn sg_reclaim_excludes_new_leases() -> Result { + let (_dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); + obj.sgt_access.store(SGT_USERS, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); + obj.sgt_access.store(0, Ordering::Release); + Ok(()) + } + + #[test] + fn sg_import_attachment_is_not_owned() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mut table = MaybeUninit::::zeroed(); + let mut attachment = MaybeUninit::::zeroed(); + // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay + // live until the lease drops and are detached before the real object destructor runs. + unsafe { + (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); + (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); + } + let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { + let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held. Capture results before restoring the fixture, so a + // failed assertion cannot leave a mock attachment for the GEM destructor. + unsafe { + let held = obj.release_idle_pages_locked(); + drop(lease); + ( + no_cleanup, + held, + obj.release_idle_pages_locked(), + obj.purge_locked(), + (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), + ) + } + }); + // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. + unsafe { + (*obj.as_raw()).import_attach = ptr::null_mut(); + (*obj.as_raw_shmem()).sgt = ptr::null_mut(); + } + let (no_cleanup, held, unleased, purge, unchanged) = result?; + assert!(no_cleanup); + assert_eq!(held, Err(EBUSY)); + assert_eq!(unleased, Err(EINVAL)); + assert_eq!(purge, Err(EBUSY)); + assert!(unchanged); + Ok(()) + } - // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it - // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device - // bound to it. + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -908,10 +1153,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + // A failed lookup must not bind the cleanup registration to the wrong device. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // TODO: Someday, we should test that creating an sg_table here still succeeds. + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); Ok(()) } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a7f..e43d5ae131c853 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,6 +151,59 @@ impl Mm { } } +/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). +/// +/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report +/// show for a process. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RssCounter { + /// Resident file-backed pages (`MM_FILEPAGES`). + File, + /// Resident anonymous pages (`MM_ANONPAGES`). + Anon, + /// Swapped-out anonymous pages (`MM_SWAPENTS`). + Swap, + /// Resident shared-memory pages (`MM_SHMEMPAGES`). + Shmem, +} + +impl RssCounter { + fn as_raw(self) -> core::ffi::c_int { + let member = match self { + Self::File => bindings::MM_FILEPAGES, + Self::Anon => bindings::MM_ANONPAGES, + Self::Swap => bindings::MM_SWAPENTS, + Self::Shmem => bindings::MM_SHMEMPAGES, + }; + member as core::ffi::c_int + } +} + +// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these +// methods are safe even if `mm_users` is zero. +impl Mm { + /// Adds `pages` (which may be negative) to one of the resident-set counters. + /// + /// This is how a driver attributes pages it holds on behalf of a process, but which are not + /// mapped into it, to that process. Every page added must be subtracted again before the + /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad + /// rss-counter state. + #[inline] + pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; + } + + /// Returns the exact current value of one of the resident-set counters. + #[inline] + pub fn rss_counter(&self, counter: RssCounter) -> usize { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } + } +} + // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer. From 1e95ba37f1ad72c42bc952e6c977147063de6152 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 03:52:09 -0400 Subject: [PATCH 02/68] drm/asahi: publish bound G17 compute work without the admission passes A compute command on a bound queue went through every admission step on each submission. ensure_compute took the device mutex for a replay and witness pass, shared_compute took a preparation lease and the mutex again, and build_compute did both once more to find its queue already bound. compute_packet then took a third lease and the mutex a fourth time, replaying compute again before publishing. Compute also never published from its submit ioctl, so even a command with nothing to wait for went through the scheduler worker first. Its completions never fed the CPU-capacity detector, so the host loop feeding a compute-bound GPU ran at the lowest CPU performance level. Check what ensure_compute establishes under the one device-mutex section that publishes, and run the admission passes only when that check fails. Publish a dependency-free compute command from its submit ioctl when its queue is bound and nothing has to wait, as renders do. Keep a separate backlog per engine so that a command waiting in one engine's scheduler does not stop the other engine's inline publication. Report compute dispatch envelopes to the detector as render passes are. (cherry picked from commit ad691cd5e9f76aa9e486d93b560c58d5c8ff88c4) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- drivers/gpu/drm/asahi/g17/feed.rs | 2 +- drivers/gpu/drm/asahi/g17/initdata.rs | 6 + drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/runtime.rs | 108 +++++++++++++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 16 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/submit.rs | 4 +- 9 files changed, 138 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4f5b87bc853dc0..dacd6de9a3a489 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -546,7 +546,13 @@ impl Completion { return; } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 0045dff413922f..760f352046192b 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -410,7 +410,7 @@ impl Feed { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 10af261aeb3acf..049c93b6a466a7 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -394,6 +394,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 61b8c3b7b47373..8ce9704d5ff1fd 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -429,7 +429,7 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 6a9b15789339fe..1f57f2913a0208 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -216,7 +216,7 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -298,8 +298,7 @@ impl Queue { checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() + let early = dependencies.is_empty() && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( id, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6f4a5355d6eb39..284c5d7bc1eb99 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -441,6 +441,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, } @@ -521,6 +522,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, @@ -1163,6 +1165,18 @@ impl super::Firmware { } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, @@ -1317,6 +1331,85 @@ impl Backend { Err(EBUSY) } + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.compute_inline(packet, &mut deferred); + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => result, + } + } + /// Called after taking this packet's publication claim. All waits and work /// state allocation precede the device mutex; only the checked epoch commits. fn compute_packet( @@ -1326,6 +1419,10 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + let mut bound = true; let mut recovery_polls = 0; let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); @@ -1341,8 +1438,10 @@ impl Backend { // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; + if !bound { + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. // Wait for one under the packet's admission deadline, as render @@ -1387,7 +1486,10 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; - if firmware.queues.compute(self.owner).is_none() { + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; drop(state); drop(lease); continue; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7d690470cd7b58..c75b525413ee87 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,14 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } @@ -274,11 +281,14 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 59db76b9f109ae..c657a45bdfa483 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -16,6 +16,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index b77fc13b7ed2fb..6c5710554e7f5e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -17,13 +17,13 @@ use kernel::{ /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the /// command instead. Never returned to userspace. -pub(super) const FALLBACK: Error = EINPROGRESS; +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { /// The submitting ioctl holds the logical queue's mutex and never sleeps /// for contention or a closed preparation gate: it returns FALLBACK, and /// the scheduler's worker publishes the command and owns the waits. - pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { if sleep { return self.shared.preparations.enter(false); } From d50ae6734304df5860414678688cebd264d040e2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 03:53:48 -0400 Subject: [PATCH 03/68] drm/asahi: wake G17 render publication when its pair is returned A render whose pair was out on loan slept 200 us before trying again. The event worker lends each pair with completions out for an off-lock observation after every completion event. At queue depths of two and more, completions and submissions interleave, so a submission often found its pair on loan for a few microseconds and then slept the full 200 us. Count the device's change notifications. Every pair return already notifies under the device mutex. Wait for the notification under that mutex, unless one arrived since the attempt started, with the same deadline as before. (cherry picked from commit 1121ae36b8d426be24c41ae360281b5f0709a69d) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/mod.rs | 15 +++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 4 ++-- drivers/gpu/drm/asahi/g17/runtime/render/grow.rs | 2 +- .../gpu/drm/asahi/g17/runtime/render/observe.rs | 2 +- .../gpu/drm/asahi/g17/runtime/render/prepare.rs | 6 +++--- .../gpu/drm/asahi/g17/runtime/render/submit.rs | 14 ++++++++++---- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2999daec47015d..46b5a2f8789073 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -55,6 +55,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU64, Ordering, // }; @@ -795,6 +796,9 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// Counts `changed` notifications, so that a waiter which checked under + /// the device mutex can tell whether it already missed one. + changes: AtomicU64, } impl_has_work! { @@ -884,6 +888,16 @@ impl Shared { drop(retired); } + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changes.fetch_add(1, Ordering::AcqRel); + self.changed.notify_all(); + } + + fn changes(&self) -> u64 { + self.changes.load(Ordering::Acquire) + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -1063,6 +1077,7 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + changes: AtomicU64::new(0), }), GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 284c5d7bc1eb99..3a496d2728f76e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -594,7 +594,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -654,7 +654,7 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index e3cb174721aaa1..20569af84bc7bf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -280,7 +280,7 @@ impl crate::g17::Shared { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 4ede20d316c648..52bdd964ca4ae7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -164,7 +164,7 @@ impl crate::g17::Shared { return Err(error); } let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); + self.note_changed(); if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e2034c31765959..04afe2e05c1079 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -426,7 +426,7 @@ impl super::super::Backend { } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -548,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -711,7 +711,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 6c5710554e7f5e..e7d0cf1168c980 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -144,7 +144,7 @@ impl Backend { /// scheduler it last ran for to be consumed and the pair parked. The event /// worker observes both, so kick it on every poll; time out only after the /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + fn wait_render_busy(&self, since: &mut Option, seen: u64, sleep: bool) -> Result { if !sleep { return Err(FALLBACK); } @@ -153,7 +153,12 @@ impl Backend { return Err(ETIMEDOUT); } self.shared.queue_events(); - fsleep(Delta::from_micros(200)); + // Every pair return notifies under the device mutex: wake on it rather + // than sleeping out a fixed interval behind a short observation loan. + let mut state = self.shared.state.lock(); + if self.shared.changes() == seen { + let _ = self.shared.changed.wait_timeout(&mut state, kernel::time::msecs_to_jiffies(1)); + } Ok(()) } @@ -194,6 +199,7 @@ impl Backend { let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); loop { + let seen = self.shared.changes(); if packet.cancel_requested() { return Err(ECANCELED); } @@ -242,7 +248,7 @@ impl Backend { } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, seen, sleep)?; continue; } result => result?, @@ -270,7 +276,7 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, seen, sleep)?; } else if !sleep { return Err(FALLBACK); } From 99a80c83155e9a1b147a750de75aee4bf6aa084f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:05:39 -0400 Subject: [PATCH 04/68] drm/asahi: clear each G17 compute descriptor only once Object::initialize already clears the complete descriptor before calling its encoder. Fill that zeroed representation directly instead of clearing the same 16 KiB again. Name the encoder write_zeroed to make its precondition explicit. The firmware bytes and validation remain unchanged. The matched module passed the targeted compute/synchronization gate and checked GPU readbacks. (cherry picked from commit abd85935bd4db7db6be23b8194b1173c9a8988c5) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 ++++---- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0cb7304e306776..b1afdad816b6b7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -507,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -522,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 713809907dd4bf..b75e00da0dad42 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1904,7 +1904,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( From bbfd3c310e702580a705a2f7dd97f2b3fb6269bb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:05:53 -0400 Subject: [PATCH 05/68] drm/asahi: retain compute pool backing through pending channel admission A publisher waiting for a physical compute channel reclaimed idle USC backing even though freeing backing cannot create another QID. The normal warm-pool limit could also evict a retired channel while another runnable publisher was waiting to reuse it. Subsequent handoffs repeatedly allocated and cleared fresh pool backing. Distinguish channel pressure from backing allocation pressure. Track unbound compute admission through publication, cancellation and error, and retain backing for the existing compute channels while that demand remains. The last publisher schedules normal idle reclamation. Allocation-pressure reclamation, the idle warm-pool limit and expiry remain in force. Expose the admission count in the capacity diagnostics. On Neo, the matched 16-client probe drops pool allocations from 4888 to 53 and median submission time from 257 to 109 us. The Asahi KUnit suites pass 67/67; the targeted compute/synchronization gate passes 116 with 18 skips. Full mustpass qualification remains pending. (cherry picked from commit 3ef3720890f8c085f18e4f5e35db6853075b121c) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/capacity.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 4 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 57 ++++++++++++++++++++++----- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index bcbbb2e7e875db..8a7f48d8a3f35b 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,7 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 46b5a2f8789073..a4f518486ef837 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -55,6 +55,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, AtomicU64, Ordering, // }; @@ -774,6 +775,8 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, #[pin] @@ -1068,6 +1071,7 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 3a496d2728f76e..beffae857a0342 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -34,6 +34,30 @@ const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -782,7 +806,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } result => result?, @@ -795,11 +819,12 @@ impl Backend { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -833,7 +858,7 @@ impl Backend { let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } result => result?, @@ -922,7 +947,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -1423,6 +1448,7 @@ impl Backend { // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. let mut bound = true; + let mut demand = None; let mut recovery_polls = 0; let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); @@ -1439,6 +1465,7 @@ impl Backend { // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); Backend::ensure_compute(self, scratch.pages(), &admission)?; admission.check()?; } @@ -1670,6 +1697,11 @@ impl super::Firmware { // retained compute pool stays warm: render and new compute need it more. let cap = if super::freelist::lower_pressure() { 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS } else { COMPUTE_WARM_CAP }; @@ -1714,10 +1746,10 @@ impl Backend { pub(super) fn wait_capacity(&self, started: u64) -> Result { let check = || self.admission_open(); let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission) + self.wait_compute_capacity(&admission, true) } - fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -1738,8 +1770,13 @@ impl Backend { } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - if !firmware.render_control_backpressured() { + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } } From cf38f6ae2a63fe66dd961ed8ba7b09e9ef151e21 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:11:30 -0400 Subject: [PATCH 06/68] dma-buf: stop polling an array after its first pending member Fence arrays require every member to signal. During syncobj timeline walks, a pending predecessor already proves that the array is incomplete, yet the lazy signaled callback polls every successor too. Return false immediately at that predecessor, keeping callback accounting and the enabled-signaling path unchanged. Add checks for out-of-order completion, avoiding successor polls, and callback installation after a partial lazy poll. Remove the obsolete signal-on-any description from the constructor documentation. (cherry picked from commit f35d7ea1b37395a9bc1241d0fafcb698e0c12d46) Signed-off-by: DjDeveloperr --- drivers/dma-buf/dma-fence-array.c | 29 ++++++----- drivers/dma-buf/st-dma-fence-unwrap.c | 75 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c index 5e10e8df372f13..64abdcc4305801 100644 --- a/drivers/dma-buf/dma-fence-array.c +++ b/drivers/dma-buf/dma-fence-array.c @@ -107,17 +107,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) unsigned int i; /* - * We need to read num_pending before checking the enable_signal bit - * to avoid racing with the enable_signaling() implementation, which - * might decrement the counter, and cause a partial check. - * atomic_read_acquire() pairs with atomic_dec_and_test() in - * dma_fence_array_enable_signaling() - * - * The !--num_pending check is here to account for the any_signaled case - * if we race with enable_signaling(), that means the !num_pending check - * in the is_signalling_enabled branch might be outdated (num_pending - * might have been decremented), but that's fine. The user will get the - * right value when testing again later. + * Read the counter before the enable bit so callback installation + * cannot make a partial count look complete. The acquire pairs with + * the counter decrements in dma_fence_array_enable_signaling(). + * If enabling races with the lazy scan, every child must still be + * observed complete before that scan can return true. */ num_pending = atomic_read_acquire(&array->num_pending); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) { @@ -126,8 +120,16 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) return false; } + /* + * Arrays require every member to complete. A pending member already + * proves the array incomplete; polling its successors only repeats work + * during timeline-chain walks. Keep the completion counter and the + * enabled-signaling path unchanged. + */ for (i = 0; i < array->num_fences; ++i) { - if (dma_fence_is_signaled(array->fences[i]) && !--num_pending) + if (!dma_fence_is_signaled(array->fences[i])) + return false; + if (!--num_pending) goto signal; } return false; @@ -256,8 +258,7 @@ EXPORT_SYMBOL(dma_fence_array_init); * and fill it with the fences it wants to add to the object. Ownership of this * array is taken and dma_fence_put() is used on each fence on release. * - * If @signal_on_any is true the fence array signals if any fence in the array - * signals, otherwise it signals when all fences in the array signal. + * The array signals when every fence in the array signals. */ struct dma_fence_array *dma_fence_array_create(int num_fences, struct dma_fence **fences, diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index 4e7ee25372ba6f..45a50cef83cc93 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -15,6 +15,7 @@ struct mock_fence { struct dma_fence base; spinlock_t lock; + atomic_t polls; }; static const char *mock_name(struct dma_fence *f) @@ -22,9 +23,18 @@ static const char *mock_name(struct dma_fence *f) return "mock"; } +static bool mock_signaled(struct dma_fence *fence) +{ + struct mock_fence *f = container_of(fence, struct mock_fence, base); + + atomic_inc(&f->polls); + return false; +} + static const struct dma_fence_ops mock_ops = { .get_driver_name = mock_name, .get_timeline_name = mock_name, + .signaled = mock_signaled, }; static struct dma_fence *__mock_fence(u64 context, u64 seqno) @@ -36,6 +46,7 @@ static struct dma_fence *__mock_fence(u64 context, u64 seqno) return NULL; spin_lock_init(&f->lock); + atomic_set(&f->polls, 0); dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno); return &f->base; @@ -584,7 +595,71 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test) dma_fence_put(f1); } +/* + * A pending all-of array needs no polls of later members. Keep its + * independent member fences so their polling and out-of-order completion + * remain observable without enabling callbacks on the array. + */ +static void test_array_pending_prefix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + struct mock_fence *last_mock; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + last_mock = container_of(last, struct mock_fence, base); + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + KUNIT_EXPECT_EQ(test, atomic_read(&last_mock->polls), 0); + dma_fence_signal(last); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(first); + KUNIT_EXPECT_TRUE(test, dma_fence_is_signaled(array)); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + +static void test_array_pending_suffix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + dma_fence_signal(first); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + /* Installing callbacks after a lazy partial poll must still account + * for the already signaled first member and wait for the last. + */ + dma_fence_enable_signaling(array); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(last); + KUNIT_EXPECT_GT(test, dma_fence_wait_timeout(array, false, HZ), 0L); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + static struct kunit_case dma_fence_unwrap_cases[] = { + KUNIT_CASE(test_array_pending_prefix), + KUNIT_CASE(test_array_pending_suffix), KUNIT_CASE(test_sanitycheck), KUNIT_CASE(test_unwrap_array), KUNIT_CASE(test_unwrap_chain), From 403c3fe1f4d505163f41b763c220dc486b3128a6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 14:38:24 -0400 Subject: [PATCH 07/68] drm/asahi: avoid unbound compute inline attempts on compute-only queues Compute-only contexts that yield their physical channel still take the device mutex in the inline submit path before falling back to admission. Track an advisory binding hint to skip that attempt. Actual publication retains the locked VM, epoch and binding checks, and scheduler admission ordering is unchanged. Mark contexts with validated rendering before any batch member is queued, so mixed-engine queues retain the original inline path. Validated with 69 Asahi KUnit cases, targeted compute/sync CTS (116 pass, 18 skip), mixed device churn and 16-client concurrent GPU readbacks. Matched 16-process quick submission time improves from 117.27 to 89.55 us; paired render follow-ups remain at control performance. (cherry picked from commit 319805fed41bb181db8e90378beaf905f38f6297) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/context.rs | 19 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 +++++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 +++ 4 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c93ab25d55ce34..445b282dcf5b64 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -114,6 +114,11 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, @@ -189,6 +194,8 @@ impl Context { scheduler, scheduler_generation, published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), @@ -203,6 +210,18 @@ impl Context { pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b75e00da0dad42..d2ec8da9c60827 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -557,6 +557,7 @@ impl Queue { let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); Ok(Self { owner: Some(owner), retained_owner: None, @@ -1274,6 +1275,7 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1284,6 +1286,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 1f57f2913a0208..96e566d3971ff5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -409,6 +409,9 @@ impl crate::queue::Queue for Queue { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + self.context.note_render(); + } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index beffae857a0342..dbae7ea2d40218 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1411,6 +1411,9 @@ impl Backend { result?; } } + if !self.context.compute_binding_hint() { + return Err(render::FALLBACK); + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() From e17b520fdd56867fc8d57c203c0cd7bd1f8c9ce0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 15:05:59 -0400 Subject: [PATCH 08/68] drm/asahi: preserve low-pressure compute inline probes Only bypass an unbound compute-only inline attempt when multiple publishers are waiting for admission. A lone admission retains the original probe and its latency behavior. Actual publication still uses the existing locked state checks. Matched quick controls retain queue signaling latency (174.1 to 174.5 us) while 16-process submission time improves from 117.3 to 90.7 us. All 20 performance rows complete successfully. Checked mixed/concurrent GPU readbacks, 69 Asahi KUnit cases and targeted compute/sync CTS also pass. (cherry picked from commit 5a4822c6fb3d1fc0a83d9b754b477bf1db35d782) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index dbae7ea2d40218..bd95148e9eaf0d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1411,7 +1411,11 @@ impl Backend { result?; } } - if !self.context.compute_binding_hint() { + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { return Err(render::FALLBACK); } let mut state = self.shared.state.lock(); From 057073a24e7eae4d87c71e9f6d2919db99e8ed6e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 20:29:12 -0400 Subject: [PATCH 09/68] drm/asahi: submit same-VM compute input waits through firmware Recognize eligible singleton compute producers through their exported aggregate fences. Order queued consumers after actual producer publication using the existing DRM scheduled fence, then validate the current physical ticket and emit firmware waits. Preserve CPU waits for unsupported dependencies and host outputs, and revalidate recovery epochs before acquiring consumer capacity. Retain source binding loans through consumer retirement, propagate source errors before dependent completion, and release failed submission pins when the last runtime owner retires even if exported fence references survive. Keep bounded member counters in byte atomics so submission state retains its original allocation class. Validated with 78 KUnit tests, the compute/synchronization CTS gate, chained readback, process exits and mixed compute/render pressure. Fixed-core ordinary compute matches the control while the queue-hop path improves. Existing intermittent unrelated-client loss during forced recovery remains unresolved. (cherry picked from commit 026065c435260844e39d05a182e99b128c5f9b2d) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/g17/capacity.rs | 4 + drivers/gpu/drm/asahi/g17/completion.rs | 32 +- drivers/gpu/drm/asahi/g17/context.rs | 1 + drivers/gpu/drm/asahi/g17/fence.rs | 366 +++++++++++++------ drivers/gpu/drm/asahi/g17/job.rs | 8 + drivers/gpu/drm/asahi/g17/mod.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 112 +++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 35 +- drivers/gpu/drm/asahi/g17/runtime.rs | 127 ++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 49 +++ drivers/gpu/drm/asahi/g17/sync_input.c | 42 +++ drivers/gpu/drm/asahi/g17/sync_input.h | 18 + drivers/gpu/drm/asahi/g17/sync_input.rs | 245 +++++++++++++ 14 files changed, 905 insertions(+), 137 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.h create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 6a5783c4c54da8..d21dfc842c4454 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o asahi-$(CONFIG_DRM_ASAHI_UAT_ADOPTION) += g15/uat_glue.o diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 8a7f48d8a3f35b..57406e5487a372 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,10 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index dacd6de9a3a489..406efc0e221fef 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -92,6 +92,8 @@ impl Destinations { Ok(Self(retained)) } + pub(crate) fn has_outputs(&self) -> bool { self.0.is_some() } + fn iter(&self) -> core::slice::Iter<'_, Option> { let destinations: &[Option] = self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); @@ -254,6 +256,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -286,6 +289,7 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute @@ -313,6 +317,7 @@ impl Completion { terminal, compute, feed, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -357,6 +362,26 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { sources.check()?; } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } @@ -525,6 +550,7 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -534,6 +560,7 @@ impl Completion { } fn release_healthy(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -541,7 +568,10 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { result = Err(error); } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 445b282dcf5b64..1818fd8c6922d8 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -354,6 +354,7 @@ pub(crate) struct WorkStateLease { word: AtomicU32, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { Self { context, diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 3414e36db9d365..25e5f6b04d5581 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,10 +7,10 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, - dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, new_mutex, prelude::*, str::CStr, @@ -48,106 +48,263 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { Ok(Fence::from_fence(&fence)) } -/// All command members of an ioctl share its work-state node and aggregate fence. +/// The existing aggregate-fence allocation owns the existing submission state. +/// Submission never owns this fence, so exported references cannot form a cycle. +pub(crate) struct ExportFence { + source: Option>, +} +static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); + +#[vtable] +impl FenceOps for ExportFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-aggregate") + } +} + +#[inline(never)] +fn export_fallback(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, ExportFence { source: None })?; + // SAFETY: Unique, initialized and still private. Only this constructor + // publishes ExportFence objects; its payload is immutable after sharing. + let ops = unsafe { + (*fence.raw()).context = bindings::dma_fence_context_alloc(1); + (*fence.raw()).ops as usize + }; + let previous = EXPORT_OPS.load(Ordering::Acquire); + if previous == 0 { + match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => {}, + Err(previous) if previous == ops => {}, + Err(_) => return Err(EIO), + } + } else if previous != ops { + return Err(EIO); + } + Ok(fence) +} + +/// Caller retains the leaf fence throughout lookup. Exact registered operations +/// identify the repr(C) FenceObject payload, not a driver or timeline name. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) + -> Option> +{ + let expected = EXPORT_OPS.load(Ordering::Acquire); + if expected == 0 { return None; } + // SAFETY: The caller owns a live fence reference. ExportFence has a unique + // timeline callback and nontrivial payload drop. No cast precedes the exact + // registered vtable check. FenceObject begins with its dma_fence. + let object = unsafe { + if (*raw).ops as usize != expected { return None; } + &*(raw as *const FenceObject) + }; + object.source.as_ref().filter(|source| source.firmware_source).cloned() +} + +struct State { + work: Option>, + ready: Option, +} + +// One Member per parsed command, plus the enqueue guard. Exported references +// retain the allocation but do not contribute to either of these counters. +kernel::static_assert!(crate::file::MAX_COMMANDS_PER_SUBMISSION < u8::MAX as u32); + /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] pub(crate) struct Submission { - fence: UserFence, - /// Successful retirement drops only this bookkeeping owner; runtime owners - /// and failed submissions keep their work-state allocation. #[pin] - work_state: Mutex>>, - remaining: AtomicU32, + state: Mutex, + remaining: AtomicU8, error: AtomicI32, + /// Live Member objects, plus the enqueue guard, independently of signalling. + owners: AtomicU8, + firmware_source: bool, } impl Submission { - fn new(fence: UserFence, context: Arc) -> Result> { - let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - fence, - work_state <- new_mutex!(Some(work_state), "G17 submission work state"), - remaining: AtomicU32::new(1), - error: AtomicI32::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) - } - - /// Add one command before enqueue ends. The parser bounds the count to 64. - pub(crate) fn member(self: &Arc) -> Member { - self.remaining.fetch_add(1, Ordering::Relaxed); - Member { - submission: self.clone(), - complete: AtomicBool::new(false), - } + fn new(context: Arc, firmware_source: bool) -> Result> { + let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Self::with_work(Some(work), firmware_source) } - - fn member_done(&self, result: Result) { - if let Err(error) = result { - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } + fn with_work(work: Option>, firmware_source: bool) -> Result> { + Arc::pin_init(pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + }), GFP_KERNEL) + } + pub(super) fn source_status(&self) -> Result { + let complete = self.remaining.load(Ordering::Acquire) == 0; + let error = self.error.load(Ordering::Acquire); + if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + Ok(complete) + } + pub(super) fn same_vm(&self, status: &Arc) -> bool { + self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } + fn set_ready(&self, ready: Fence) { + if !self.firmware_source { return; } + let mut state = self.state.lock(); + if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + } + fn note_error(&self, error: Error) -> bool { + self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + } + fn release_state(&self) { + let old = { + let mut state = self.state.lock(); + (state.work.take(), state.ready.take()) + }; + drop(old); + } + fn owner_done(&self) { + // Every hardware owner retains its Completion/Member through exact + // retirement or stop. Exported fence FDs must not retain failed pins. + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + } + fn member_done(&self, fence: &UserFence, result: Result) { + if let Err(error) = result { self.note_error(error); } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { - self.fence.set_error(Error::from_errno(error)); - } else { - // Scheduler cleanup can wait behind submission on the same worker. - // Once every member and enqueue have succeeded, retained packets - // must not prevent new submissions from acquiring work-state nodes. - let retired = self.work_state.lock().take(); - drop(retired); - } - self.fence.signal(); + if error != 0 { fence.set_error(Error::from_errno(error)); } + else { self.release_state(); } + fence.signal(); } } +} +#[derive(Clone)] +pub(crate) struct Aggregate(UserFence); +impl Aggregate { + fn source(&self) -> &Arc { + // Only Outputs::publish constructs Aggregate, after setting the payload. + self.0.source.as_ref().expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { Fence::from_fence(&self.0) } + pub(crate) fn member(&self) -> Member { + self.source().remaining.fetch_add(1, Ordering::Relaxed); + self.source().owners.fetch_add(1, Ordering::Relaxed); + Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + } fn finish_enqueue(&self, result: Result) { - // A dropped, never-enqueued member reports cancellation. The ioctl's actual - // failure takes precedence, while published members still retain their work. - if let Err(error) = result { - self.error.store(error.to_errno(), Ordering::Release); - } - self.member_done(result); + if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + self.source().member_done(&self.0, result); + self.source().owner_done(); } } -/// One command's contribution to its submission, including construction failure. pub(crate) struct Member { - submission: Arc, + aggregate: Aggregate, complete: AtomicBool, } - impl Member { + pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } pub(crate) fn work_state(&self) -> Result> { - self.submission - .work_state - .lock() - .as_ref() - .cloned() - .ok_or(ECANCELED) + self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) } - + pub(super) fn is_source(&self, source: &Arc) -> bool { + self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } + pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.submission.member_done(result); + self.aggregate.source().member_done(&self.aggregate.0, result); } } } +#[kunit_tests(asahi_g17_aggregate_promise)] +mod tests { + use super::*; + use kernel::sync::LockClassKey; + static KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + + fn contexts() -> FenceContexts { + FenceContexts::new(1, c_str!("asahi_promise_test"), KEY).unwrap() + } + fn aggregate(contexts: &FenceContexts) -> Aggregate { + let mut fence = export_fallback(contexts).unwrap(); + fence.source = Some(Submission::with_work(None, true).unwrap()); + Aggregate(fence.into()) + } + + #[test] + fn exact_export_identity_recognizes_unpublished_source() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let plain = independent(&contexts).unwrap(); + // SAFETY: Both references remain owned throughout the checked lookup. + assert!(unsafe { exported_source_raw(plain.raw()) }.is_none()); + let source = unsafe { exported_source_raw(aggregate.0.raw()) }.unwrap(); + assert!(Arc::ptr_eq(&source, aggregate.source())); + assert_eq!(source.source_status().unwrap(), false); + aggregate.finish_enqueue(Ok(())); + assert_eq!(source.source_status().unwrap(), true); + } + + #[test] + fn failed_export_releases_state_after_last_runtime_owner() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let source = aggregate.source().clone(); + source.state.lock().ready = Some(Fence::from_fence(&independent(&contexts).unwrap())); + let member = aggregate.member(); + aggregate.finish_enqueue(Ok(())); + member.complete(Err(EIO)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert!(source.state.lock().ready.is_some()); + drop(member); + // Keep both the exported fence and source state alive, as userspace can. + assert!(source.state.lock().ready.is_none()); + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } + + #[test] + fn actual_enqueue_error_wins_over_dropped_member_cancellation() { + let aggregate = aggregate(&contexts()); + drop(aggregate.member()); + aggregate.finish_enqueue(Err(ENOMEM)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + } + + #[test] + fn rejected_private_fence_never_exports_source_state() { + let fence = export_fallback(&contexts()).unwrap(); + fence.set_error(ENOMEM); + fence.signal(); + // SAFETY: The unique fence owns its initialized allocation. + assert!(unsafe { exported_source_raw(fence.raw()) }.is_none()); + } + + #[test] + fn maximum_submission_keeps_all_members_and_enqueue_guard() { + let aggregate = aggregate(&contexts()); + let source = aggregate.source().clone(); + let mut members: [Option; crate::file::MAX_COMMANDS_PER_SUBMISSION as usize] = + core::array::from_fn(|_| Some(aggregate.member())); + assert_eq!(source.remaining.load(Ordering::Acquire), 65); + assert_eq!(source.owners.load(Ordering::Acquire), 65); + aggregate.finish_enqueue(Ok(())); + for member in &members { + assert_eq!(super::super::job::fence_status(&aggregate.fence()), 0); + member.as_ref().unwrap().complete(Ok(())); + } + assert!(super::super::job::fence_status(&aggregate.fence()) > 0); + assert_eq!(source.remaining.load(Ordering::Acquire), 0); + for member in &mut members { drop(member.take()); } + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } +} impl Drop for Member { fn drop(&mut self) { self.complete(Err(ECANCELED)); + self.aggregate.source().owner_done(); } } @@ -155,83 +312,54 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(fence)); - } + } else { sync.syncobj.replace_fence(Some(fence)); } } } -/// Keeps resolved output syncobjs reachable on every fallible submission path. -/// Input syncobjs stay in the same vector, avoiding a separate output allocation. pub(crate) struct Outputs { - fallback: UserFence, + fallback: Option>, syncs: KVec, input_count: usize, - submission: Option>, + submission: Option, result: Result, } - impl Outputs { - pub(crate) fn new( - contexts: &FenceContexts, - mut syncs: KVec, - input_count: usize, - allocation_failure: &Fence, - ) -> Result { - if input_count > syncs.len() { - return Err(EINVAL); - } - let fallback = match independent(contexts) { + pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, + input_count: usize, allocation_failure: &Fence) -> Result + { + if input_count > syncs.len() { return Err(EINVAL); } + let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - // Syncobjs and timeline-chain storage are already resolved. - // No work has been accepted; publish the pre-signaled ENOMEM - // fence without another allocation before returning the error. install(syncs.drain(input_count..), allocation_failure); return Err(error); } }; - Ok(Self { - fallback, - syncs, - input_count, - submission: None, - result: Err(ECANCELED), - }) - } - - pub(crate) fn inputs(&self) -> &[SyncItem] { - &self.syncs[..self.input_count] + Ok(Self { fallback: Some(fallback), syncs, input_count, + submission: None, result: Err(ECANCELED) }) } - - /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, context: Arc) -> Result> { - let submission = Submission::new(self.fallback.clone(), context)?; + pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } + pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + let source = Submission::new(context, firmware_source)?; + let mut fence = self.fallback.take().ok_or(EINVAL)?; + fence.source = Some(source); + let submission = Aggregate(fence.into()); install(self.syncs.drain(self.input_count..), &submission.fence()); self.submission = Some(submission.clone()); Ok(submission) } - - /// The caller first records terminal VM errors and settles prior failed work. - pub(crate) fn finish(mut self, result: Result) -> Result { - self.result = result; - result - } + pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } } - impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { submission.finish_enqueue(self.result); return; } - if let Err(error) = self.result { - self.fallback.set_error(error); + if let Some(fallback) = self.fallback.take() { + if let Err(error) = self.result { fallback.set_error(error); } + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + fallback.signal(); } - install( - self.syncs.drain(self.input_count..), - &Fence::from_fence(&self.fallback), - ); - self.fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 8ce9704d5ff1fd..a36eab9761867d 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -150,6 +150,7 @@ impl Packet { super::dependency::replay_status(fence_status(fence))?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -183,6 +184,7 @@ impl Packet { for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -262,6 +264,9 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } @@ -430,6 +435,9 @@ impl Scheduler { // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. if !(early && self.backend.try_early(&packet)) { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a4f518486ef837..2a580a4d3341b2 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -27,6 +27,7 @@ mod data_segment; mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod sync_input; mod event; mod feed; mod fence; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index d2ec8da9c60827..85c65a9b22d536 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -538,6 +538,8 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -598,6 +600,8 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), }) } pub(crate) fn qid(&self) -> u8 { @@ -1231,11 +1235,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1343,7 +1351,9 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1780,6 +1790,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1814,11 +1835,22 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { return Err(EAGAIN); } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1928,6 +1960,13 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence); + let address = self.graph.record(record_slot, 0x900 + index * 0x60, 0x60, record)?; + wait_vas.push(address, GFP_KERNEL)?; + } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1962,15 +2001,22 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - Err(error) - })?; + })().or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; } + Err(error) + })?; visible = true; self.unpublished_binding = false; let qos = host.qos_publish( @@ -2282,6 +2328,7 @@ impl Queue { impl Queue { pub(crate) fn can_rearm(&self) -> bool { self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } } @@ -2323,3 +2370,50 @@ impl Queue { Ok((false, retired)) } } + + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source(&self, source: &Arc, + status: &Arc, epoch: u64) -> bool + { + !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source)) + } + + pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, + status: &Arc, epoch: u64) -> Result> + { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { continue; } + if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } + if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { return Err(EAGAIN); } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { return Err(ECANCELED); } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some((active.published, dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), hold))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 96e566d3971ff5..777c4bc1440acd 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -179,7 +179,7 @@ impl Queue { command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -198,6 +198,22 @@ impl Queue { let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + let sources = if engine == Engine::Compute && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[0], *sequence) + || job::fence_status(&fences.completed) > 0) + && self.frontiers[1].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || job::fence_status(&fences.completed) > 0) + && self.failures.iter().enumerate().all(|(index, failure)| + failure.as_ref().is_none_or(|(sequence, _)| + !Order::contains(order.wait_through[index], *sequence))) + { self.backend.input_sources(inputs) } else { None }; + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] @@ -217,6 +233,7 @@ impl Queue { engine == Engine::Compute, host_predecessors.clone().next().is_some(), self.backend.feed(), + sources.clone(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -294,10 +311,19 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } + if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } let early = dependencies.is_empty() && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( @@ -416,7 +442,10 @@ impl crate::queue::Queue for Queue { for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index bd95148e9eaf0d..d0f3f3cfb2eb26 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -959,6 +959,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -994,6 +995,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1051,6 +1053,12 @@ impl DeferredBatch { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1062,11 +1070,29 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1140,11 +1166,75 @@ impl compute::Host for ComputeHost<'_> { } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + merged.insert(sequence, dependency)?; + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { @@ -1183,9 +1273,12 @@ impl super::Firmware { accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } @@ -1442,6 +1535,27 @@ impl Backend { } } + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) + } + /// Called after taking this packet's publication claim. All waits and work /// state allocation precede the device mutex; only the checked epoch commits. fn compute_packet( @@ -1468,6 +1582,11 @@ impl Backend { let admission = Admission::new(deadline, &check); loop { admission.check()?; + if !self.sources_published(packet)? { + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index c75b525413ee87..b393afd9c8df29 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -112,6 +112,55 @@ impl Backend { } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c new file mode 100644 index 00000000000000..e6b91988c3c287 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "sync_input.h" + +/* Called without the device mutex: status polling may run fence callbacks. */ +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity) +{ + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + size_t count = 0; + int error = 0; + + dma_fence_unwrap_for_each(leaf, &cursor, head) { + if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + break; + } + if (count == capacity) { + error = -ENOSPC; + break; + } + out[count].fence = dma_fence_get(leaf); + out[count].status = dma_fence_get_status(leaf); + ++count; + } + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + if (error) { + while (count) + dma_fence_put(out[--count].fence); + return error; + } + return count; +} + +void asahi_g17_put_fence(struct dma_fence *fence) +{ + dma_fence_put(fence); +} diff --git a/drivers/gpu/drm/asahi/g17/sync_input.h b/drivers/gpu/drm/asahi/g17/sync_input.h new file mode 100644 index 00000000000000..2b6e80f4b7f668 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_INPUT_H +#define ASAHI_G17_SYNC_INPUT_H + +#include + +struct dma_fence; + +struct asahi_g17_fence_leaf { + struct dma_fence *fence; + int status; +}; + +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity); +void asahi_g17_put_fence(struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_input.rs b/drivers/gpu/drm/asahi/g17/sync_input.rs new file mode 100644 index 00000000000000..1cd9a9ba761aa3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded input-fence components. Collection happens before the device mutex, +//! because polling a container's status can signal it and run callbacks. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +#[repr(C)] +pub(super) struct Leaf { + pub(super) fence: *mut bindings::dma_fence, + pub(super) status: i32, +} + +unsafe extern "C" { + fn asahi_g17_collect_fences(head: *mut bindings::dma_fence, + out: *mut Leaf, capacity: usize) -> i32; + fn asahi_g17_put_fence(fence: *mut bindings::dma_fence); +} + +impl Drop for Leaf { + fn drop(&mut self) { + // SAFETY: Successful collection transfers exactly one owned reference. + unsafe { asahi_g17_put_fence(self.fence) }; + } +} + +/// Allocation, unsupported containers, or a large prefix retain normal waits. +pub(super) fn collect(input: &Fence) -> Option> { + let mut capacity = 4; + loop { + let mut leaves = KVec::with_capacity(capacity, GFP_KERNEL).ok()?; + // SAFETY: The input owns its fence. The C helper writes at most capacity + // fully initialized records and releases every reference on failure. + let count = unsafe { + asahi_g17_collect_fences(input.raw(), leaves.as_mut_ptr(), capacity) + }; + if count == ENOSPC.to_errno() && capacity < 128 { + capacity = (capacity * 4).min(128); + continue; + } + if count < 0 { return None; } + // SAFETY: The helper returned this many initialized, owned records. + unsafe { leaves.set_len(count as usize) }; + return Some(leaves); + } +} + + +use super::fence::Submission; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{new_mutex, sync::{Arc, Mutex}}; + +pub(super) struct Source { + pub(super) submission: Arc, +} + +/// The counter is independent of a physical queue's allocation and survives +/// processor stop until the last consumer releases its binding loan. +pub(super) struct Hold(Arc); +impl Hold { + fn new(readers: &Arc) -> Result { + readers.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |count| count.checked_add(1)).map_err(|_| EOVERFLOW)?; + Ok(Self(readers.clone())) + } +} +impl Drop for Hold { + fn drop(&mut self) { self.0.fetch_sub(1, Ordering::AcqRel); } +} + +/// A new owner starts with no loans. Acquiring a loan requires exclusive access +/// and arms the hint; eligibility queries stay read-only. A validated owner +/// transition can clear the hint after all independent loans have been released. +#[repr(transparent)] +pub(super) struct ReaderHint { + possible: bool, +} +impl ReaderHint { + pub(super) const fn new() -> Self { + Self { possible: false } + } + pub(super) fn hold(&mut self, count: &Arc) -> Result { + let hold = Hold::new(count)?; + self.possible = true; + Ok(hold) + } + pub(super) fn has_readers(&self, count: &AtomicU32) -> bool { + self.possible && count.load(Ordering::Acquire) != 0 + } + pub(super) fn reset_for_owner(&mut self, count: &AtomicU32) -> Result { + if self.has_readers(count) { return Err(EBUSY); } + self.possible = false; + Ok(()) + } +} + +#[pin_data] +pub(super) struct Sources { + inputs: KVec, + publication_waits: KVec, + pub(super) sources: KVec, + #[pin] + holds: Mutex>, +} +impl Sources { + pub(super) fn new(inputs: KVec, sources: KVec, + publication_waits: KVec) -> Result> { + CONVERTED.fetch_add(1, Ordering::Relaxed); + if !publication_waits.is_empty() { PENDING.fetch_add(1, Ordering::Relaxed); } + Arc::pin_init(pin_init!(Self { + inputs, sources, publication_waits, + holds <- new_mutex!(KVec::new(), "G17 source binding loans"), + }), GFP_KERNEL) + } + pub(super) fn contains_input(&self, input: &Fence) -> bool { + self.inputs.iter().any(|fence| fence.raw() == input.raw()) + } + pub(super) fn publication_waits(&self) -> &[Fence] { &self.publication_waits } + pub(super) fn check(&self) -> Result { + for source in &self.sources { source.submission.source_status()?; } + Ok(()) + } + pub(super) fn set_holds(&self, holds: KVec) { + let old = core::mem::replace(&mut *self.holds.lock(), holds); + drop(old); + } + pub(super) fn release(&self) { self.set_holds(KVec::new()); } +} + + +static CONVERTED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PUBLISHED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PENDING: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn pending_count() -> u64 { PENDING.load(Ordering::Relaxed) } +pub(super) fn note_published() { PUBLISHED.fetch_add(1, Ordering::Relaxed); } +pub(super) fn counters() -> (u64, u64) { + (CONVERTED.load(Ordering::Relaxed), PUBLISHED.load(Ordering::Relaxed)) +} + +/// A revalidated compute completion dependency for a direct work descriptor. +#[derive(Clone, Copy)] +pub(super) struct NativeWait { + pub(super) sequence: u64, + pub(super) dependency: super::fw::kick::KickDependency, + pub(super) stamp_va: u64, + pub(super) event_slot: u32, +} +impl NativeWait { + pub(super) fn bytes(&self, consumer_stamp: u32, uuid: u64) -> [u8; 0x48] { + let mut out = [0; 0x48]; + let timestamp = self.dependency.word() & ((1u64 << 40) - 1); + for (offset, value) in [ + (0, 4u32), + (0x14, u32::from(self.dependency.qid())), + (0x18, super::fw::stamp_value(timestamp as u32)), + (0x24, self.event_slot), + (0x28, consumer_stamp), + (0x34, 1), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(4, self.stamp_va), (0xc, timestamp), (0x2c, uuid)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + out + } +} + +static NATIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn note_native(count: usize) { NATIVE.fetch_add(count as u64, Ordering::Relaxed); } +pub(super) fn native_count() -> u64 { NATIVE.load(Ordering::Relaxed) } + +#[kunit_tests(asahi_g17_input_wait)] +mod tests { + use super::*; + use crate::g17::fw::kick::{KickDependency, KickTimestamp}; + + #[test] + fn completion_record_preserves_distinct_event_and_queue_domains() { + let wait = NativeWait { + sequence: 1, + dependency: KickDependency::new(37, + KickTimestamp::new(0x12_3456_789a).unwrap()).unwrap(), + stamp_va: 0x1122_3344_5566_7788, + event_slot: 7, + }; + // Protocol fixture includes the unaligned stamp pointer, full 40-bit + // timestamp, wrapped 24-bit stamp and independent legacy event slot. + let expected: [u8; 0x48] = [ + 4, 0, 0, 0, 0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11, 0x9a, 0x78, 0x56, 0x34, + 0x12, 0, 0, 0, 37, 0, 0, 0, + 0, 0x9a, 0x78, 0x56, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 0xcc, 0xbb, 0xaa, 8, 7, 6, 5, + 4, 3, 2, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + ]; + assert_eq!(wait.bytes(0xaabb_cc00, 0x0102_0304_0506_0708), expected); + } + + #[test] + fn failed_loan_acquisition_never_wraps_or_releases_another_reader() { + let readers = Arc::new(AtomicU32::new(u32::MAX - 1), GFP_KERNEL).unwrap(); + let last = Hold::new(&readers).unwrap(); + assert!(matches!(Hold::new(&readers), Err(error) if error == EOVERFLOW)); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX); + drop(last); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX - 1); + } + + #[test] + fn loan_outlives_original_queue_counter_reference() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + drop(readers); + assert_eq!(second.0.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(second.0.load(Ordering::Acquire), 1); + drop(second); + } + + #[test] + fn reader_guard_tracks_partial_release_and_rearms_after_zero() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + assert!(!hint.has_readers(&readers)); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + assert!(matches!(hint.reset_for_owner(&readers), Err(error) if error == EBUSY)); + drop(first); + assert!(hint.has_readers(&readers)); + drop(second); + assert!(!hint.has_readers(&readers)); + hint.reset_for_owner(&readers).unwrap(); + let next = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + drop(next); + assert!(!hint.has_readers(&readers)); + } +} From 734eace5754db1471df68843951156ce14f83b0d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:29:21 -0400 Subject: [PATCH 10/68] drm/asahi: treat a firmware source as complete once its retirement is queued Compute retirement removes the producer from its queue's active list under the device mutex, but signals the producer's fence only after the lock is dropped. A consumer published in between found neither an active producer nor a completed one, fell back to the scheduler, and every later command in the chain then waited for its producer's publication through the scheduler. Mark the singleton firmware source retired as its successful retirement is queued (errors are already recorded there), after rechecking its own inputs. sync.signal queue-to-queue (timeline, 2 queues): scheduler fallbacks for chain consumers 57% -> 0.1%. (cherry picked from commit a3be7f63500c76d117000ee1474713e27889aabe) (cherry picked from commit 89c8873651b729fdd4e981e87b9d0df0d0ad0017) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 7 +++++++ drivers/gpu/drm/asahi/g17/fence.rs | 12 +++++++++++- drivers/gpu/drm/asahi/g17/runtime.rs | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 406efc0e221fef..5936bcb1b933a0 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -377,6 +377,13 @@ impl Completion { pub(super) fn note_input_error(&self, error: Error) -> bool { self.output.output().member.note_error(error) } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { self.note_input_error(error); } + } + } pub(super) fn check_source_inputs(&self) -> Result { if let Some(sources) = &self.sources { sources.check()?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 25e5f6b04d5581..da883cede52df7 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -121,6 +121,10 @@ pub(crate) struct Submission { /// Live Member objects, plus the enqueue guard, independently of signalling. owners: AtomicU8, firmware_source: bool, + /// Set under the device mutex when the only member's successful retirement + /// is queued, before its fence is signalled off-lock: a consumer published + /// in between needs no wait on it. + retired: AtomicBool, } impl Submission { @@ -135,10 +139,12 @@ impl Submission { error: AtomicI32::new(0), owners: AtomicU8::new(1), firmware_source, + retired: AtomicBool::new(false), }), GFP_KERNEL) } pub(super) fn source_status(&self) -> Result { - let complete = self.remaining.load(Ordering::Acquire) == 0; + let complete = self.remaining.load(Ordering::Acquire) == 0 + || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } Ok(complete) @@ -212,6 +218,10 @@ impl Member { } pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn note_retired(&self) { + let source = self.aggregate.source(); + if source.firmware_source { source.retired.store(true, Ordering::Release); } + } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { self.aggregate.source().member_done(&self.aggregate.0, result); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d0f3f3cfb2eb26..736fed78b0cc18 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1057,6 +1057,8 @@ impl DeferredBatch { Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), _ => {}, } self.values.push_within_capacity(value).map_err(|_| EIO)?; From 072782a9b2cdfd5a938299358cb61e1702a3a40f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:29:21 -0400 Subject: [PATCH 11/68] drm/asahi: publish G17 commands from the ioctl at any queue depth Early publication was limited to queues with fewer than two outstanding commands, so every deeper queue paid a scheduler worker hop per command and cross-queue chains advanced one scheduler round trip per link. The backend already falls back to the scheduler behind any backlog and on every capacity miss, which preserves submission order, so the depth limit only added latency. sync.signal queue-to-queue (timeline, 2 queues) 94.1 -> 56.2 us/hop together with the preceding change; compute and render depth rows unchanged. (cherry picked from commit 8508510641edcedb5ae1417f560cdff4da646bda) (cherry picked from commit 3ff4f4baa05a88d5f99d2fee76089cf72b5617a4) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 777c4bc1440acd..c959f8bc8c4b89 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -27,7 +27,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -324,8 +323,10 @@ impl Queue { checked_inputs.push(ready.clone(), GFP_KERNEL)?; } } - let early = dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), From 7c1b71c2c29a049e85bfb33654ff2fa97757e250 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:49:26 -0400 Subject: [PATCH 12/68] drm/asahi: map G17 compute descriptors cached for the firmware Render descriptors already come from the retained object pool with a firmware-cached mapping; compute descriptors were mapped uncached, so the firmware read every compute descriptor from memory on each command. Allocate the compute descriptor ring from the same pool (fully cached prefix, retained until processor stop) and give the hardware an uncached canonical low alias for the descriptor address it fetches. compute.light d64 27.4 -> 24.3 us/submit, queue-to-queue hop 58.1 -> 54.8 us; compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit c85c050ae2ef77cbeebddb70c6602db477675090) (cherry picked from commit a0dfc3c0b5a4fced66119fded07ed141990a9025) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 46 +++++++++++++--------- drivers/gpu/drm/asahi/g17/runtime.rs | 9 +++-- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 85c65a9b22d536..3547f312224b4f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -15,7 +15,7 @@ use crate::g17::{ queue::*, variant::PointerLayout, }, - object::{Allocator, CpuMap, KernelObject}, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -83,24 +83,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -127,6 +135,7 @@ struct Graph { impl Graph { fn new( alloc: &Allocator<'_>, + objects: &Arc, context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, @@ -134,7 +143,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; @@ -250,12 +259,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -547,6 +556,7 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, @@ -554,7 +564,7 @@ impl Queue { operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 736fed78b0cc18..d94e344c6f7137 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -790,7 +790,7 @@ impl Backend { admission.check()?; let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -814,7 +814,7 @@ impl Backend { match rebound { Rebind::Done => return Ok(()), Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, @@ -838,7 +838,7 @@ impl Backend { } Err(error) => return Err(error), }; - (firmware.uat.clone(), Ok(reservation)) + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -853,7 +853,7 @@ impl Backend { // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc)?)) + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { Err(error) if error == ENOSPC => { @@ -908,6 +908,7 @@ impl Backend { self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new( &alloc, + &objects, backing, self.owner, self.context.clone(), From 501f3f850c7d1b63594188fd55fbefd6006cd64f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:38:10 -0400 Subject: [PATCH 13/68] drm/asahi: size the G17 compute QID budget by open logical queues The compute channel budget was static: compute could only take a QID while every one of the 58 possible execution contexts could still install a render pair, which always left exactly ten compute channels. A client without a channel then made every idle channel of every live client shareable before binding one, so sixteen compute processes evicted each other on nearly every submission (map/unmap and TLB invalidation churn, ComputeReserve refused 52k times in one benchmark pass). Installed QIDs never change engine before processor stop, so a compute channel still permanently narrows render. Size the render reserve by the logical queues that are actually open plus 24 queues of headroom, and refuse a new logical queue only when its dedicated render pair would no longer fit. With every context open this is the previous rule. Rebinding now prefers a channel of a closed owner, then a vacant one to re-arm, then a fresh QID, and only then shares the least recently used idle channel of a single live client. gpubench on Neo (us/submit, matched baseline): compute.light procs d16 92.0 -> 33.4; compute.light queue rows unchanged (d64 27.4 -> 26.5). Stress gate passes. (cherry picked from commit bf7ae5bf04b7d3f14b5ec94ef1f8c1434b4fe494) (cherry picked from commit a36e352d8564947071fef7e11dba709f399cd106) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/kick.rs | 121 ++++++++++++++------- drivers/gpu/drm/asahi/g17/queue/compute.rs | 24 +++- drivers/gpu/drm/asahi/g17/runtime.rs | 95 +++++++++++++--- 3 files changed, 178 insertions(+), 62 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index c337a8b298f193..87a90516d1b42f 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -24,22 +24,38 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -// Protect one tiling/fragment pair for each logical execution context, plus -// the two unpublished bootstrap reservations. The remaining ten compute -// channels are shared after exact retirement, rather than pinned per client. -const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; -/// Logical queues that can each hold a render pair: what the render reserve leaves beside the -/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone -/// (every execution context with no floor). +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT)] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. pub(super) const fn queue_limit(floor: usize) -> u32 { - ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 } const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - assert!(COMPUTE_CHANNELS >= 4); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -85,7 +101,7 @@ impl QueueIds { publication: 0, leased: 0, // The reserve must keep render capacity above the floor. - floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -113,6 +129,53 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -129,10 +192,8 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never the host's; the reserve shrinks - // with them, so the compute channel count stays the same. + // The QIDs below the floor are never available to the host. let total = QID_COUNT - self.floor; - let reserve = COMPUTE_RENDER_RESERVE - self.floor; let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() @@ -145,27 +206,9 @@ impl QueueIds { } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if total - self.leased + replacement - <= reserve.saturating_sub(render - replacement) - { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); - } + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred .map(|index| usize::from(index) + self.floor) @@ -249,10 +292,10 @@ mod accounting_tests { #[test] fn render_identities_satisfy_the_compute_reserve() { let mut ids = table(); - for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + for owner in 1..=FULL_RENDER_RESERVE as u64 { ids.reserve(owner, DataMaster::Tiling, None).unwrap(); } - for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.leased, QID_COUNT); @@ -263,14 +306,14 @@ mod accounting_tests { #[test] fn compute_cannot_consume_the_render_minimum() { let mut ids = table(); - for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); } #[test] diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3547f312224b4f..69841e5cd2aebb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -534,6 +534,7 @@ pub(crate) struct Queue { published_config: Option, ordinal: u32, submitted: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -597,6 +598,7 @@ impl Queue { published_config: None, ordinal: 0, submitted: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -1926,6 +1928,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -2323,16 +2326,27 @@ impl Queue { /// Logical queue/context ownership is unchanged. The existing binding is /// retained through the successor's exact replacement witness. pub(crate) fn share_idle(&mut self) -> Result { - if !self.installed || self.owner.is_none() || !self.retirement_ready - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.previous.is_some() || self.pool_phase != PoolPhase::Populated - { + if !self.shareable() { return Ok(false); } self.release_owner()?; Ok(self.reusable()) } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.installed && self.owner.is_some() && self.retirement_ready + && !self.growing && !self.quarantined && !self.awaiting_witness + && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 + && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } } impl Queue { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d94e344c6f7137..44770b0cb30417 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -155,12 +155,43 @@ impl Registry { Rebind::Fresh }); } - // Physical channels are a device budget, not a permanent entitlement - // of each open logical queue. Only exact idle retirement can yield one. - for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); + } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .min_by_key(|queue| (!queue.pool_fits(required), queue.last_published())) + .map(|queue| queue.qid()); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)?; + if !queue.share_idle()? { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; for queue in self @@ -183,9 +214,9 @@ impl Registry { } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -352,9 +383,10 @@ impl Registry { .ok_or_else(|| self.compute_admission_error( crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { @@ -491,6 +523,11 @@ impl Backend { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( @@ -1861,8 +1898,9 @@ impl Registry { self.compute.iter().flatten().count(), self.compute.len(), self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} total={}", - self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { queue.write_capacity(f)?; @@ -1893,10 +1931,6 @@ impl Backend { if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { return Ok(()); } - for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; - } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; // Pool eviction addresses backing allocation pressure, not channel @@ -1937,12 +1971,17 @@ mod shared_capacity_tests { ) .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - // Physical compute channels are device-owned and bounded independently - // of how many logical clients have queued unsignalled work. - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); @@ -1994,11 +2033,31 @@ mod shared_capacity_tests { leases.push(lease, GFP_KERNEL).unwrap(); } } - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert_eq!(leases.len(), 36); } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } } impl Backend { From d8ef9cd59ff67aced753d4440b2d7413f245a707 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:40:28 -0400 Subject: [PATCH 14/68] drm/asahi: order same-VM render attachment overlaps in firmware A render whose attachment spans overlapped a live render of another logical queue in the same VM waited in admission, on the host, until that render retired. gpubench render.light queues renders every queue into one image, so all queues ran one render at a time: run_job blocked for (N-1) renders and the 2/4/8-queue rows stayed at one serialized step (174/201/217 us/submit). Order such overlaps at publication instead. The device mutex already orders publications; the later render's fragment kick now waits in firmware for the newest published overlapping fragment kick of every other pair, so its tiling kick stays concurrent and no host wait or scheduler stall is involved. Published QIDs keep their kick rings, so these stamps only advance. An admission bound on other pairs of the VM keeps the barrier count within its fixed room. Cross-VM overlaps through shared objects keep the host wait, so one VM's fault can never hold another VM's kick in firmware. render.light queues d2/d4/d8: 173.8/201.3/216.9 -> 81.4/66.9/62.1 us/submit. Other rows unchanged; the mild stress gate (CTS compute/sync, cross-queue order/data, slow-read pressure, mixed churn) passes. (cherry picked from commit 809d95cfdd685b6f94570cd313b1723c7f01f466) (cherry picked from commit b141ca9b16c33abb26238782cd7c0b751b5f52fe) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 7 ++ drivers/gpu/drm/asahi/g17/hazard.rs | 46 ++++++++++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 78 ++++++++++++++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 17 +++- .../drm/asahi/g17/runtime/render/submit.rs | 10 ++- 5 files changed, 150 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 5936bcb1b933a0..d692a98acea8c9 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -547,6 +547,13 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 20daaab85b8325..4d9e99d9a9cc0c 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -3,11 +3,21 @@ //! Attachment overlap ordering across physical render pairs and shared images. //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. +//! +//! Within one VM, a render of another logical owner is ordered at publication: +//! the device mutex totally orders publications, and the later render's +//! fragment kick waits in firmware for the newest overlapping fragment kick of +//! every other pair. Admission never waits for such an overlap. Only a +//! cross-VM overlap through a shared object waits on the host for retirement, +//! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; +/// Firmware barriers one render publication may add for same-VM overlaps, +/// one per other pair. +pub(crate) const PUBLICATION_BARRIERS: usize = 7; #[derive(Clone, Copy)] struct Span { @@ -56,7 +66,12 @@ impl Footprint { self.shared |= self.count != before; } - fn overlaps(&self, other: &Self) -> bool { + /// Whether a publication orders an overlap with `other` in firmware. + pub(crate) fn same_vm(&self, other: &Self) -> bool { + self.vm == other.vm + } + + pub(crate) fn overlaps(&self, other: &Self) -> bool { if self.vm != other.vm && !(self.shared && other.shared) { return false; } @@ -86,6 +101,13 @@ pub(crate) struct Live<'a> { /// A queue prefix already orders covered predecessors. Pending arrivals order /// different owners fairly without imposing order on unrelated attachments. +/// +/// A same-VM overlap with another owner's admitted render needs no wait; its +/// publication carries one barrier per other pair. That count stays within +/// [`PUBLICATION_BARRIERS`] because such a render is admitted only while fewer +/// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: +/// of any set of renders overlapping one publication, the last admitted saw all +/// the others live. pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -94,10 +116,29 @@ pub(crate) fn plan<'a>( live: impl IntoIterator>, ) -> Plan { let mut selected = None; + let mut ordered = false; + let mut pairs = [0u8; PUBLICATION_BARRIERS]; + let mut count = 0; + let mut crowded = false; for entry in live { + let foreign = entry.owner != owner && footprint.same_vm(entry.footprint); + if let Some(slot) = entry.slot.filter(|_| foreign) { + if !pairs[..count].contains(&slot) { + if count == PUBLICATION_BARRIERS { + crowded = true; + } else { + pairs[count] = slot; + count += 1; + } + } + } if entry.quarantined || !footprint.overlaps(entry.footprint) { continue; } + if foreign { + ordered |= entry.slot.is_some(); + continue; + } let Some(slot) = entry.slot else { if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { return Plan::Wait; @@ -116,5 +157,8 @@ pub(crate) fn plan<'a>( Some(_) => return Plan::Wait, } } + if ordered && crowded { + return Plan::Wait; + } selected.map_or(Plan::Any, Plan::Pair) } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ebdc79d2b2dd3d..7533d345cfa6d4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,6 +44,9 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; @@ -332,6 +336,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -339,6 +344,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -705,13 +716,72 @@ impl RenderLease { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fb6c77fbdf04f2..dfe3426eb6c24c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -484,7 +489,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -495,7 +500,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -521,6 +526,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -840,6 +850,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index e7d0cf1168c980..663e62e8f86fca 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -462,7 +462,15 @@ impl crate::g17::Firmware { { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; // An owner change taken from a live owner which closed meanwhile: its // first command waits until that scheduler's release has been consumed. From 04f31452142d5a8bd0f3b4920297010bc3efab63 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:01:13 -0400 Subject: [PATCH 15/68] drm/asahi: map G17 compute command records cached for the firmware Each compute command writes a queue configuration record, an optional predecessor and completion-wait records, and an announcement; the firmware read all of them through an uncached mapping. Render keeps its equivalent configuration records cached and only its announcements uncached. Move the configuration, predecessor and completion-wait records into a retained pool object with a firmware-cached mapping at the same per-slot offsets; announcements stay uncached. compute.light d1 161.0 -> 152.2 us, d64 24.3 -> 23.1 us/submit, queue-to-queue hop 54.8 -> 51.2 us; compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit b3d5b9d6294bb46d63778264205da9d3b750651e) (cherry picked from commit a475f4f9f1f721b6636a4147dacbcf9f6224763b) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 69841e5cd2aebb..625f65e6d67721 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -123,7 +123,11 @@ struct Graph { graph: KernelObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, /// Host words of the pointer block in this firmware build. @@ -171,6 +175,14 @@ impl Graph { let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * RECORD_STRIDE, + RECORD_SLOTS * RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -181,6 +193,7 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, @@ -1729,7 +1742,12 @@ impl Graph { if slot >= RECORD_SLOTS { return Err(EINVAL); } - (&mut self.records, slot * RECORD_STRIDE + offset) + let object: &mut KernelObject = if offset == ANNOUNCE { + &mut self.records + } else { + &mut self.cached_records + }; + (object, slot * RECORD_STRIDE + offset) } else { (&mut self.graph, 0x6800 + offset) }; From a2c5a171b9bfc0b012970e4b13cd39c247550ceb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:03:59 -0400 Subject: [PATCH 16/68] drm/asahi: stop the G17 compute completion scan once every record is found Every event pass with an unobserved compute command walked all 2048 records of the compute completion ring under the device mutex to find the few it needed; that walk was half of the event worker's CPU time. Resume each scan after the last record the previous scan needed and stop as soon as every requested identity has an observation. A scan that finds nothing still wraps the whole ring once, as before. scan_compute leaves the top of the 16-process profile; benchmark rows are unchanged within noise (they are not bound by the scan). Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 67a49c9c40b31b96a7834c8c185dc20e0bac1fa2) (cherry picked from commit ca71e4449b4a23d5f4e0149db76cf5198bfa7e5f) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/initdata.rs | 21 +++++++++++++------ .../drm/asahi/g17/queue/compute/retirement.rs | 12 +++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 15 +++++++++++-- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 049c93b6a466a7..b9a8f326d970d2 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -679,19 +679,28 @@ impl InitData { /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. + /// Visits ring slots from `start`, wrapping once, until `record` reports + /// that nothing more is wanted. Returns the slot after the last one visited + /// when the scan stopped early, or `start` after a complete wrap. pub(super) fn scan_compute_completions( &self, - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + start: usize, + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { let object = self.object(self.completion_rings[0])?; + let slots = (abi::COMPLETION_RING_SIZE / 0x40).min(0x800); + let start = start % slots; fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + for step in 0..slots { + let slot = (start + step) % slots; + if record(ComputeCompletion { object, offset: slot * 0x40, - })?; + })? { + return Ok((slot + 1) % slots); + } } - Ok(()) + Ok(start) } fn object(&self, index: usize) -> Result<&KernelObject> { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index e938ec566dbacb..655e2921f25828 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -174,6 +174,8 @@ pub(crate) struct Batch { requests: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, } impl Batch { @@ -182,12 +184,18 @@ impl Batch { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, slots: [(0, 0); QIDS], open: None, + unobserved: 0, }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 } pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { @@ -214,6 +222,7 @@ impl Batch { .map_err(|_| ENOSPC)?; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; Ok(()) } /// Capture the control word once, then read only a requested identity's payload. @@ -247,6 +256,9 @@ impl Batch { { return Ok(()); } + if request.observed.is_none() { + self.unobserved -= 1; + } request.observed = Some(Observation { timestamps }); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 44770b0cb30417..745cd62cae595d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -86,6 +86,8 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -121,6 +123,7 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -1403,9 +1406,17 @@ impl super::Firmware { { requested |= queue.request_observations(&mut registry.observations, masks)?; } + // Completion records are appended in order: resume after the last + // record a previous scan needed and stop once every request is found. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + registry.completion_cursor = self.init.scan_compute_completions( + registry.completion_cursor, + |record| { + observations.fold(record)?; + Ok(observations.complete()) + }, + )?; } let view = qos::View::new(self.init.qos()?)?; for queue in registry From 8525488940d884fe894dd65d982d636ef5f9849d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:06:56 -0400 Subject: [PATCH 17/68] drm/asahi: publish G17 compute into a bound queue without the device mutex Every compute publication took the device mutex, which the event worker also holds for each service pass, and repeated device-wide work on every submit (replay scan, render-prefix scan, binding checks). Give each physical compute queue its own lock (a cell the registry owns; device-mutex holders lock cells one at a time) and move the device-wide state a publication needs into a per-session port that is reached under a preparation lease: compute work-ring slot reservations, QoS accounting under its own spinlock, the recovery generation, the full-effort flag and the doorbell, whose RTKit borrows are serialized by a mailbox lock. The steady-state path (bound, installed, no input sources, no render prefix) publishes from the submit ioctl or run_job with only the queue's lock and the port's small locks; everything else falls back to the device-mutex path, which now reserves work-ring slots through the same port. A command published while the event worker is scanning is not part of that scan's completion snapshot and is left unobserved until the next one. gpubench on Neo (us/submit, vs matched baseline): compute.light d1 166.3 -> 156.2, d64 27.4 -> 25.8, procs d16 33.4 -> 33.0 (firmware per command time now bounds these rows). Stress gate passes. (cherry picked from commit 8fc91b2117de925e362342a842b0d0befbba5312) (cherry picked from commit 1cdac9ba0dddec96f59eb02a5067ee343aae43f4) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/context.rs | 4 + drivers/gpu/drm/asahi/g17/initdata.rs | 63 ++- drivers/gpu/drm/asahi/g17/kick.rs | 9 - drivers/gpu/drm/asahi/g17/mod.rs | 112 ++++- drivers/gpu/drm/asahi/g17/port.rs | 267 ++++++++++++ drivers/gpu/drm/asahi/g17/power.rs | 2 +- drivers/gpu/drm/asahi/g17/qos.rs | 19 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 + .../drm/asahi/g17/queue/compute/retirement.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 386 ++++++++++++++---- .../drm/asahi/g17/runtime/compute_control.rs | 13 +- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 28 +- .../drm/asahi/g17/runtime/compute_replay.rs | 22 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 12 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 21 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- 16 files changed, 784 insertions(+), 188 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/port.rs diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 1818fd8c6922d8..564d761d37f974 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -214,6 +214,10 @@ impl Context { self.render_seen.load(Ordering::Acquire) || self.compute_binding_hint.load(Ordering::Relaxed) } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } pub(crate) fn note_render(&self) { if !self.render_seen.load(Ordering::Relaxed) { self.render_seen.store(true, Ordering::Release); diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index b9a8f326d970d2..6d0b2f9951956e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -129,14 +129,6 @@ impl WorkPublication<'_> { ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -516,7 +508,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -569,16 +561,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -677,6 +659,49 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every compute work + /// ring, the QoS table and the recovery state word. They stay valid for this session. + pub(super) fn port_views( + &self, + mailbox: *const super::Mailbox, + ) -> Result { + use super::fw::{channels, queue::{DataMaster, Policy}}; + let ring = |priority: u8| -> Result { + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = |offset: usize| -> Result<*const AtomicU32> { + Ok(state.word(base + offset)?) + }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize + * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let mut compute = [ring(0)?; Policy::PRIORITIES as usize]; + for (priority, view) in compute.iter_mut().enumerate().skip(1) { + *view = ring(priority as u8)?; + } + Ok(super::port::Views { + compute, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. /// Visits ring slots from `start`, wrapping once, until `record` reports diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 87a90516d1b42f..f3183bb005df60 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -87,7 +87,6 @@ struct Identity { pub(crate) struct QueueIds { entries: [Option; QID_COUNT], initial_render: [Option; 2], - publication: u64, leased: usize, /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). floor: usize, @@ -98,7 +97,6 @@ impl QueueIds { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], - publication: 0, leased: 0, // The reserve must keep render capacity above the floor. floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, @@ -239,13 +237,6 @@ impl QueueIds { Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. Compute and - /// render publications share it, so a watchdog can order work across both engines. - pub(crate) fn next_publication(&mut self) -> Result { - self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2a580a4d3341b2..8ade9af88c154b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -40,6 +40,7 @@ mod job; mod capacity; mod kick; mod object; +mod port; pub(crate) mod power; mod preparation; mod qos; @@ -397,13 +398,48 @@ impl rtkit::Operations for RoleOps { } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, _crash_buffer: Option, } @@ -458,7 +494,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -487,19 +523,20 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() + self.rtkit.with(|rtkit| rtkit.boot()) } /// Starts the firmware endpoints and hands the firmware its initialization root. An /// acknowledgment message is awaited here; one in status A is the caller's to poll. fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; match ack { fw::variant::InitAck::Message => self.state.wait_for_ack(), fw::variant::InitAck::Status => Ok(()), @@ -507,8 +544,8 @@ impl Coprocessor { } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -575,6 +612,10 @@ struct Firmware { secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, @@ -649,12 +690,13 @@ impl Firmware { } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { @@ -662,21 +704,35 @@ impl Firmware { } init.publish_effort(effort)?; state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let effort = match self.effort.idle(now_ns, in_flight) { + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; None } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), }) @@ -803,6 +859,8 @@ struct Shared { /// Counts `changed` notifications, so that a waiter which checked under /// the device mutex can tell whether it already missed one. changes: AtomicU64, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -902,6 +960,20 @@ impl Shared { self.changes.load(Ordering::Acquire) } + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -1083,6 +1155,7 @@ impl Gpu { state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), changes: AtomicU64::new(0), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; @@ -1105,6 +1178,9 @@ impl Gpu { GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) @@ -1216,6 +1292,8 @@ impl Gpu { secondary, primary, bridge, + port: port::Port::new()?, + seen_submissions: 0, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), @@ -1262,6 +1340,8 @@ impl Gpu { if let Some(root_entries) = root_entries { session.uat.confirm_firmware_root_entries(root_entries)?; } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach(session.init.port_views(mailbox)?)?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs new file mode 100644 index 00000000000000..63eab753632be7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide publication state that physical queues share without the device mutex. +//! +//! A firmware session owns its port, and the views it holds name that session's +//! firmware-shared memory. Code that does not hold the device mutex reaches the port only +//! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every +//! lease before they detach the session. Each piece below is atomic or has its own lock. +//! +//! Lock order: device mutex -> compute queue cell -> the locks of this port. + +use super::{ + fw::{ + channels::{WorkSlot, WORK_RING_SLOTS}, + queue::{DataMaster, Policy}, + }, + qos, +}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{ + new_spinlock, + prelude::*, + sync::{Arc, SetOnce, SpinLock}, +}; + +const PRIORITIES: usize = Policy::PRIORITIES as usize; + +/// One priority class's compute work ring: its slot array and the cursors of its two firmware +/// consumers and of the host producer. +#[derive(Clone, Copy)] +pub(super) struct RingView { + pub(super) slots: *mut u8, + pub(super) consumers: [*const AtomicU32; 2], + pub(super) producer: *const AtomicU32, +} + +impl RingView { + fn cursors(&self) -> Result<[u32; 3]> { + // SAFETY: The views name 32-bit words of the session's retained firmware objects, which + // the port's owner keeps alive. Other agents only use single 32-bit accesses. + let values = unsafe { + [ + (*self.consumers[0]).load(Ordering::Relaxed), + (*self.consumers[1]).load(Ordering::Relaxed), + (*self.producer).load(Ordering::Relaxed), + ] + }; + if values.iter().any(|&value| value >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(values) + } + + /// Slots the host may still fill before reaching either consumer. + fn free(&self) -> Result { + let [first, second, producer] = self.cursors()?; + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + Ok(room(first).min(room(second))) + } +} + +/// Raw views of the session's firmware-shared memory, fixed once the firmware is built. +pub(super) struct Views { + pub(super) compute: [RingView; PRIORITIES], + pub(super) qos: *mut u8, + pub(super) recovery: *const AtomicU32, + pub(super) mailbox: *const super::Mailbox, +} + +// SAFETY: The views only name memory and a mailbox owned by the firmware session, which outlives +// every user of the port. Accesses are atomic words or serialized by the port's locks. +unsafe impl Send for Views {} +// SAFETY: As above. +unsafe impl Sync for Views {} + +#[pin_data] +pub(super) struct Port { + views: SetOnce, + /// Compute work-ring slots reserved by publishers that have not exposed them yet. + #[pin] + rings: SpinLock<[u32; PRIORITIES]>, + #[pin] + qos: SpinLock, + /// The accepted recovery generation, mirrored for publishers without the device mutex. + generation: AtomicU64, + /// Device publication order shared by compute and render commands. + publications: AtomicU64, + /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. + effort_full: AtomicBool, + submissions: AtomicU64, +} + +/// A compute work-ring slot reserved before any command becomes visible. Dropping an unused +/// claim returns the slot. +pub(super) struct OuterClaim<'a> { + port: &'a Port, + priority: usize, + armed: bool, +} + +impl Drop for OuterClaim<'_> { + fn drop(&mut self) { + if self.armed { + let mut reserved = self.port.rings.lock(); + reserved[self.priority] = reserved[self.priority].saturating_sub(1); + } + } +} + +impl Port { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + views: SetOnce::new(), + rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), + qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + generation: AtomicU64::new(0), + publications: AtomicU64::new(0), + effort_full: AtomicBool::new(false), + submissions: AtomicU64::new(0), + }), + GFP_KERNEL, + ) + } + + /// Called once while the session is built, before any publisher can see the port. + pub(super) fn attach(&self, views: Views) -> Result { + if self.views.populate(views) { + Ok(()) + } else { + Err(EBUSY) + } + } + + fn views(&self) -> Result<&Views> { + self.views.as_ref().ok_or(ENODEV) + } + + fn ring(&self, priority: u8) -> Result<(usize, RingView)> { + let index = usize::from(priority); + Ok((index, *self.views()?.compute.get(index).ok_or(EINVAL)?)) + } + + /// Reserves one slot of the compute work ring of `priority`. EAGAIN while firmware has not + /// consumed enough slots; only consumption, never a waiting publisher, frees one. + pub(super) fn reserve_compute(&self, priority: u8) -> Result> { + let (index, ring) = self.ring(priority)?; + let mut reserved = self.rings.lock(); + if ring.free()? <= reserved[index] { + return Err(EAGAIN); + } + reserved[index] += 1; + Ok(OuterClaim { + port: self, + priority: index, + armed: true, + }) + } + + /// Whether a compute publication at `priority` can reserve a slot now. + pub(super) fn compute_ready(&self, priority: u8) -> Result { + let (index, ring) = self.ring(priority)?; + let reserved = self.rings.lock(); + Ok(ring.free()? > reserved[index]) + } + + /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns + /// the new producer, the retained consumer witness for this slot. + pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { + return Err(EINVAL); + } + let ring = self.views()?.compute[claim.priority]; + let mut reserved = self.rings.lock(); + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EIO); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records, checked when the session built + // it; `producer` is in range. The reservation proves firmware does not own this slot and + // the ring lock excludes every other host writer. The record is a padding-free ABI value. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + ring.slots.add(producer as usize * core::mem::size_of::()), + core::mem::size_of::(), + ); + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; + fence(Ordering::SeqCst); + reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + claim.armed = false; + Ok(next) + } + + fn qos_view(&self) -> Result> { + // SAFETY: The session retains the QoS table for at least as long as this port. + unsafe { qos::View::from_raw(self.views()?.qos) } + } + + pub(super) fn qos_publish( + &self, + owner: qos::Owner, + scheduler: u64, + policy: Policy, + ) -> Result { + let view = self.qos_view()?; + self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + } + + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { + let view = self.qos_view()?; + self.qos.lock().cancel(&view, publication) + } + + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { + let view = self.qos_view()?; + self.qos.lock().complete(&view, owners) + } + + /// Rings the primary coprocessor's doorbell. + pub(super) fn notify(&self, message: u64) -> Result { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. + let mailbox = unsafe { &*self.views()?.mailbox }; + mailbox.send(super::EP_DOORBELL, message) + } + + pub(super) fn recovery_state(&self) -> Result { + let word = self.views()?.recovery; + fence(Ordering::Acquire); + // SAFETY: As for `RingView::cursors`. + Ok(unsafe { (*word).load(Ordering::Relaxed) }) + } + + pub(super) fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Called with the device mutex held after the accepted generation changed, before + /// publication reopens. + pub(super) fn set_generation(&self, generation: u64) { + self.generation.store(generation, Ordering::Release); + } + + pub(super) fn next_publication(&self) -> u64 { + self.publications.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// A publisher without the device mutex counts itself before it checks that full effort is + /// applied. The idle clear withdraws the flag before it rechecks the count, so either the + /// clear sees this submission or this submission sees the flag withdrawn. + pub(super) fn note_submission(&self) -> bool { + self.submissions.fetch_add(1, Ordering::SeqCst); + self.effort_full.load(Ordering::SeqCst) + } + + pub(super) fn submissions(&self) -> u64 { + self.submissions.load(Ordering::SeqCst) + } + + pub(super) fn set_effort_full(&self, full: bool) { + self.effort_full.store(full, Ordering::SeqCst); + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 7fef15084c50a0..b0e21e70bbe18e 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -6,7 +6,7 @@ use core::sync::atomic::AtomicU32; pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after /// the last job. Not measured on other firmware; at or below it the hold stays off. diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 9a100db4cffdfc..4f6913ad5ee95c 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -2,17 +2,14 @@ //! Shared QoS publication and exact-completion accounting for every engine. //! -//! The device publication lock serializes this host state. Publishing identity +//! The port's QoS lock serializes this host state. Publishing identity //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{ - fw::{qos as fw, queue::Policy}, - object::KernelObject, -}; +use super::fw::{qos as fw, queue::Policy}; use crate::hw::t8140::qos as cfg; use core::{ - mem::{offset_of, size_of}, + mem::offset_of, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; @@ -30,15 +27,17 @@ pub(crate) struct View<'a> { times_and_counts: &'a [AtomicU64; 7], } impl<'a> View<'a> { - pub(crate) fn new(object: &'a KernelObject) -> Result { - let ptr = object.pointer(0, size_of::())?; + /// # Safety + /// + /// `ptr` must span a complete QoS table that stays mapped for `'a`. + pub(crate) unsafe fn from_raw(ptr: *mut u8) -> Result { if ptr as usize & 7 != 0 { return Err(EINVAL); } // SAFETY: pointer spans the complete layout, whose asserted offsets // and size bound every array below. Integer atomics have their integer - // representation and alignment. The retained object borrow keeps every - // shared reference alive; accesses use each field's fixed width. + // representation and alignment. The caller keeps every shared + // reference alive; accesses use each field's fixed width. Ok(unsafe { Self { rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 625f65e6d67721..a21d1c915d0c8f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2351,6 +2351,11 @@ impl Queue { Ok(self.reusable()) } + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { self.installed && self.owner.is_some() && self.retirement_ready diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 655e2921f25828..b268c1c40de00c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -262,16 +262,17 @@ impl Batch { request.observed = Some(Observation { timestamps }); Ok(()) } + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 745cd62cae595d..237c92db48ce24 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -69,13 +69,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -106,7 +196,6 @@ impl Registry { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, @@ -129,7 +218,7 @@ impl Registry { }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -137,12 +226,16 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a @@ -175,18 +268,20 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.shareable()) .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .min_by_key(|queue| (!queue.pool_fits(required), queue.last_published())) - .map(|queue| queue.qid()); + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); let Some(qid) = idle else { return Err(ENOSPC); }; admission.check()?; - let queue = self + let shared = self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid) - .ok_or(EIO)?; - if !queue.share_idle()? { + .ok_or(EIO)? + .share_idle()?; + if !shared { return Err(ENOSPC); } Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) @@ -197,7 +292,7 @@ impl Registry { fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -230,7 +325,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -242,7 +337,7 @@ impl Registry { pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -310,7 +405,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -340,7 +435,7 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -371,7 +466,7 @@ impl Registry { || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -394,7 +489,7 @@ impl Registry { let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -426,7 +521,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -441,13 +536,14 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; } @@ -495,6 +591,7 @@ mod uninstalled_cleanup_tests { /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, @@ -502,6 +599,11 @@ pub(super) struct Backend { backlog: core::sync::atomic::AtomicU32, compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, } impl Backend { @@ -580,15 +682,16 @@ impl Backend { } drop(state); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), - }, + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + }), GFP_KERNEL, )?, slot, @@ -762,7 +865,7 @@ impl Backend { if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -793,8 +896,9 @@ impl Backend { if let Some(firmware) = state.as_deref_mut() { for index in 0..firmware.queues.compute.len() { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(queue) = entry.queue.as_deref_mut() else { continue; }; - if queue.abort_growth(token)? == Some(true) { + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { // Keep the empty entry and QID reserved until all // graph aliases have been destroyed outside the lock. retired = entry.queue.take().map(|queue| (entry.reservation, queue)); @@ -1156,15 +1260,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1181,30 +1285,66 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + self.port.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.port.notify(message) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } @@ -1238,7 +1378,7 @@ impl super::Firmware { if source.submission.source_status()? { continue; } let found = self.queues.compute.iter_mut().flatten() .filter_map(|entry| entry.queue.as_deref_mut()) - .find_map(|queue| match queue.source_dependency(source, packet.completion.status(), + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), self.recovery.generation()) { Ok(None) => None, result => Some(result), @@ -1284,15 +1424,10 @@ impl super::Firmware { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() @@ -1310,10 +1445,10 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; if native.is_empty() { @@ -1341,10 +1476,11 @@ impl Registry { fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -1356,10 +1492,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1371,7 +1506,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -1418,8 +1553,8 @@ impl super::Firmware { }, )?; } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + for mut queue in registry .compute .iter_mut() .flatten() @@ -1430,7 +1565,7 @@ impl super::Firmware { masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -1440,16 +1575,16 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -1457,7 +1592,7 @@ impl super::Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -1480,7 +1615,7 @@ impl Backend { firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; @@ -1500,6 +1635,78 @@ impl Backend { Err(EBUSY) } + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + /// Publishes a dependency-free compute command from its submit ioctl when /// its queue is already bound and nothing has to wait, as renders do. Any /// wait leaves the command, unpublished, to its scheduler. @@ -1523,7 +1730,10 @@ impl Backend { return true; } }; - let result = self.compute_inline(packet, &mut deferred); + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { deferred.finish(); packet.abandon_publication(); @@ -1536,6 +1746,21 @@ impl Backend { true } + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + /// compute_packet's publication section without its waits: every condition /// it would sleep or retry for returns FALLBACK instead. fn compute_inline( @@ -1582,7 +1807,12 @@ impl Backend { Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { Err(render::FALLBACK) } - result => result, + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } } } @@ -1617,6 +1847,9 @@ impl Backend { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. let mut bound = true; @@ -1730,6 +1963,9 @@ impl Backend { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } + if result.is_ok() { + self.remember_direct(&firmware.queues); + } return result; } } @@ -1741,7 +1977,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1797,6 +2033,7 @@ impl super::Shared { dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); return; } + self.forget_port(); let firmware = self.state.lock().take(); if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } } @@ -1810,6 +2047,7 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -1849,7 +2087,7 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + for mut queue in self .queues .compute .iter_mut() diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index c407d9576e9c47..4f519743aed530 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,11 +20,10 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; let computes = firmware.queues.compute.iter().flatten() @@ -64,7 +63,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -91,7 +90,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 984b6c1f643ebc..69bc7550fc4e92 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -19,7 +19,7 @@ impl super::Registry { /// batch cannot overflow a separate cache. No allocation or wait is needed. pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { for entry in self.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { if queue.acknowledge_exit(cookie) { return; } @@ -31,12 +31,12 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.settle_exit( recovery_closed, - &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -48,7 +48,7 @@ impl Firmware { // Publication has already retained each kill. Retry only the doorbell; // failed attempts do not extend its bounded polling window. for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); } } @@ -85,14 +85,16 @@ impl Firmware { if self.render_control_backpressured() { return Ok(()); } - let queue = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .ok_or(EIO)?; - let Some(owner) = queue.exit_owner(context) else { - return Ok(()); + let (owner, target) = { + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + (owner, queue.qid()) }; - let target = queue.qid(); let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { @@ -130,7 +132,7 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d71f410c76fb48..0a8ebc28c02e29 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -90,7 +84,7 @@ impl Firmware { let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4fd6e567419860..e46c820efa1550 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -225,9 +225,9 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -235,7 +235,7 @@ impl Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -276,7 +276,11 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service.firmware.port.set_generation(service.firmware.recovery.generation()); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index c657a45bdfa483..2877dea91d0ea7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -514,21 +514,10 @@ impl crate::g17::queue::render::Host for Host<'_> { scheduler: u64, policy: crate::g17::fw::queue::Policy, ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.firmware.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration @@ -559,7 +548,7 @@ impl crate::g17::queue::render::Host for Host<'_> { self.firmware.queues.qids.publish(id) } fn next_publication(&mut self) -> Result { - self.firmware.queues.qids.next_publication() + Ok(self.firmware.port.next_publication()) } fn publish_outer_pair( &mut self, @@ -619,13 +608,11 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; let next = queues.render.pools.completed(pool_id, generation)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f8020e8962f849..aaa2e281c69eef 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -115,7 +115,7 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) + .filter_map(|entry| entry.queue.as_deref()) .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { From 9fbb8dea7e409effc1d4cff600211def784d4371 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:21:42 -0400 Subject: [PATCH 18/68] drm/asahi: send a G17 compute queue configuration only when it changes Render emits its queue configuration record only for an install, a context or QoS update, or when the configuration differs from the last one the firmware consumed in the current recovery epoch. Compute appended the record to every command, so the firmware reapplied unchanged queue state each time. Use the same rule for compute, caching the last consumed configuration with its epoch and forgetting it when the item ring is reset. compute.light d2 61.9 -> 51.5, d8 25.9 -> 24.3, d64 23.9 -> 22.8 us/submit; queue-to-queue hop 51.1 -> 49.6 us. Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 0e59b1f92effb58c87f705907959de45abe77623) (cherry picked from commit 7242bf87efd2b3f4f8bf2a31fadb79b3e687f195) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 36 +++++++++++++------ drivers/gpu/drm/asahi/g17/queue/render.rs | 2 +- .../drm/asahi/g17/queue/render/publication.rs | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index a21d1c915d0c8f..5e6f20a3da43a0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -545,6 +545,9 @@ pub(crate) struct Queue { /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, last_published: u64, @@ -609,6 +612,7 @@ impl Queue { installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, last_published: 0, @@ -1436,6 +1440,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1925,6 +1930,8 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1989,7 +1996,11 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { @@ -2023,7 +2034,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -2060,14 +2071,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -2101,6 +2114,9 @@ impl Queue { self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index dfe3426eb6c24c..e063ef3a047fc7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 9f4cb05b6bedeb..c1985c72f6ec1f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 From f6055a5156cecc36a0ee5e82cdb8847565f5829e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:28:39 -0400 Subject: [PATCH 19/68] drm/asahi: add G17 resource wait domains and use them for queue creation Waits for shared capacity sampled a condition and slept 1 ms before checking again. Add availability domains: a waiter samples the domain's generation before its failed check and registers a fence against that sample (or retries when a release intervened); the release that provides the resource bumps the generation and signals only that domain's waiters. The fences can serve as scheduler prepare_job dependencies or be waited on directly. The preparation gate can carry a domain that its reopening, failure and removal signal. Logical-queue creation is the first user: a creator refused for want of a queue slot or an execution context held by a closing queue now waits for the returned slot or the reclaimed context instead of sleeping in 1 ms steps, under the same overall deadline. Tested on Neo as part of compute-wait2 (stress gate passes). (cherry picked from commit 79cf4be7732d549b0f5d1f903bed7effdb80a4fb) (cherry picked from commit 014605a3679581351913c3ab137653297f7a0cf7) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/availability.rs | 107 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 20 +++- drivers/gpu/drm/asahi/g17/preparation.rs | 28 ++++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 13 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 20 +++- 5 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/availability.rs diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs new file mode 100644 index 00000000000000..5b5d1d639a18ee --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resource-specific scheduler waits. A caller samples a domain before a failed try, then +//! registers against that sample after rolling back unpublished claims; the release that frees +//! the resource bumps the generation and signals exactly the waiters of its own domain. +//! Successful acquisition never allocates a fence, and a release without waiters costs one +//! lock round trip. + +use super::fence::{self, CompletionFence}; +use kernel::{ + c_str, + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +struct Entry { + fence: UserFence, + next: Option>, +} + +struct State { + generation: u64, + waiters: Option>, +} + +#[pin_data] +pub(crate) struct Availability { + #[pin] + state: Mutex, + fences: FenceContexts, +} + +impl Availability { + pub(crate) fn new() -> Result> { + Arc::pin_init( + try_pin_init!(Self { + state <- new_mutex!( + State { generation: 0, waiters: None }, + "g17::Availability" + ), + fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + }), + GFP_KERNEL, + ) + } + + /// Sample before checking capacity under the resource's own synchronization. + pub(crate) fn observe(&self) -> u64 { + self.state.lock().generation + } + + /// A fence the next notification signals. None means capacity changed after the sample: + /// retry instead of waiting. Called with no resource lock held. The recheck after allocation + /// closes both the failed-check/registration and the fence-allocation/release races. A + /// waiter that goes away leaves its entry for the next notification to signal. + pub(crate) fn wait(&self, observed: u64) -> Result> { + if self.observe() != observed { + return Ok(None); + } + let fence = fence::independent(&self.fences)?; + let mut entry = KBox::new( + Entry { fence: fence.clone(), next: None }, + GFP_KERNEL, + )?; + let mut state = self.state.lock(); + if state.generation != observed { + return Ok(None); + } + entry.next = state.waiters.take(); + state.waiters = Some(entry); + Ok(Some(Fence::from_fence(&fence))) + } + + /// Blocks until a notification after `observed`, for at most `timeout_ms`. For callers + /// outside the scheduler that own no other wait. + pub(crate) fn block(&self, observed: u64, timeout_ms: u32, interruptible: bool) -> Result { + let Some(fence) = self.wait(observed)? else { return Ok(()); }; + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + interruptible, + kernel::time::msecs_to_jiffies(timeout_ms) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + Ok(()) + } + + /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence + /// callbacks run outside the domain lock. + pub(crate) fn notify(&self) { + let mut waiters = { + let mut state = self.state.lock(); + state.generation = state.generation.wrapping_add(1); + state.waiters.take() + }; + while let Some(mut entry) = waiters { + waiters = entry.next.take(); + entry.fence.signal(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 8ade9af88c154b..49e18fda438f45 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -17,6 +17,7 @@ //! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state //! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +mod availability; mod buffer; mod channel; mod command; @@ -832,6 +833,8 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, /// Unbound compute publishers which can reuse a retired channel. compute_demand: AtomicU32, #[pin] @@ -1002,6 +1005,7 @@ impl Shared { // Context release returns its execution root off-lock. The next // device pass can now detach idle compute aliases and their QoS // leases, without depending on a new client's submission/event. + self.queues.notify(); self.queue_events(); } } @@ -1114,6 +1118,7 @@ impl Gpu { }; let max_frequency_khz = Self::max_frequency_khz(dev)?; let preparations = preparation::Gate::new()?; + let queues = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1144,6 +1149,7 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + queues, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1300,11 +1306,12 @@ impl Gpu { hold: power::Hold::new(), registration_power_release_pending: false, variant: shared.variant, - queues <- runtime::Registry::new( + queues <- runtime::Registry::with_waiters( shared.clusters, shared.deflake, shared.descriptor_flags, shared.variant, + Some(shared.queues.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, @@ -1575,16 +1582,21 @@ impl gpu::Gpu for Gpu { let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, Err(error) if error == ENOSPC => { // Closed logical queues can still own execution/QoS IDs // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. - if now_ns().saturating_sub(admission_started) - >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + let waited = now_ns().saturating_sub(admission_started); + let limit = job::TIMEOUT_MS as u64 * 1_000_000; + if waited >= limit { return Err(EBUSY); } self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A returned queue slot or reclaimed context wakes this. + self.shared + .queues + .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; } result => break result?, } diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0297c619b4f933..e014d4bbb6542b 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -55,20 +55,36 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -168,14 +184,18 @@ impl Gate { pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } /// Runtime failure/recovery must not join an indefinitely stalled builder. @@ -199,7 +219,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -213,6 +233,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 7533d345cfa6d4..ff2b294b6f35c6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -377,9 +377,18 @@ pub(crate) struct Pool { queues: AtomicU32, /// Logical queues admitted at most: each must be able to hold a render pair. max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, } impl Pool { + #[cfg(CONFIG_KUNIT)] pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); } @@ -389,6 +398,7 @@ impl Pool { released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), max_queues, + waiters, }), GFP_KERNEL, ) @@ -667,6 +677,9 @@ pub(crate) struct QueueSlot { impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 237c92db48ce24..88a3330155cac2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -185,11 +185,23 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT)] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { @@ -198,7 +210,7 @@ impl Registry { buffers <- BufferIds::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -621,6 +633,7 @@ impl Backend { const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; let started = now_ns(); let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -660,7 +673,10 @@ impl Backend { return Err(ERESTARTSYS); } shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; } Err(error) => return Err(error), } From ff778361a002e928cddd72145060fb20f9a685ac Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:30:03 -0400 Subject: [PATCH 20/68] drm/asahi: wait for G17 compute capacity on the release that frees it Compute admission and publication retried with sleeps: 1 ms for an unpublished input source, a 20 us..1 ms backoff for a work state, up to 200 x 1 ms for recovery, 1 ms per ring backpressure retry, 200 x 1 ms for a spared drain witness, and 1 ms per growth and channel capacity retry. Use two availability domains for compute admission; the release that provides the resource signals only the waiters of its own domain: - channels: a queue becomes reusable, shareable or re-armable (checked on each transition in the retirement scan and pool service), a scheduler release is consumed, an owner closes, or an unpublished slot and QID are returned; - progress: any compute retirement, finished replays, pool release observation, and the preparation gate reopening after recovery. prepare_job now returns the channel, work-state or recovery fence a compute packet still lacks, so run_job finds its capacity ready, and a prepared packet claims its bound channel so the channel is not shared away before run_job. Retries that remain inside publication block on the same fences; the admission slice only bounds cancellation checks. Each event pass also rechecks the level while waiters exist, which covers capacity that appears through render (QIDs, work states, closed queues). gpubench on Neo (us/submit, matched baseline): unchanged steady state (procs d16 33.1, d64 25.9, d1 162); procs d2 93.2 -> 69.2. With the channel budget forced back to ten channels (diagnostic build), sixteen processes now share them at 51.9 us/submit instead of the old 92.0, with no stall. Stress gate passes, also under the forced shortage. (cherry picked from commit 25fb5800d97db7457f093ff41206047fbd9bddcc) (cherry picked from commit 64f717dcfd1e4aa5856882181f5c2ac040c5ce4e) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/availability.rs | 4 + drivers/gpu/drm/asahi/g17/job.rs | 4 +- drivers/gpu/drm/asahi/g17/mod.rs | 24 +- drivers/gpu/drm/asahi/g17/preparation.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime.rs | 314 +++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 18 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 9 +- 9 files changed, 337 insertions(+), 63 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index 5b5d1d639a18ee..8014266f0b0ab1 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -51,6 +51,10 @@ impl Availability { self.state.lock().generation } + pub(crate) fn has_waiters(&self) -> bool { + self.state.lock().waiters.is_some() + } + /// A fence the next notification signals. None means capacity changed after the sample: /// retry instead of waiting. Called with no resource lock held. The recheck after allocation /// closes both the failed-check/registration and the fence-allocation/release races. A diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index a36eab9761867d..4c8546e60c217c 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -282,7 +282,9 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 49e18fda438f45..2f7cdf3ef81760 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,10 +12,14 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. mod availability; mod buffer; @@ -833,6 +837,12 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, /// Logical-queue creation waiting for a returned queue slot or execution context. queues: Arc, /// Unbound compute publishers which can reuse a retired channel. @@ -1117,8 +1127,10 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1149,6 +1161,8 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, queues, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index e014d4bbb6542b..0df2c5ae2ebb25 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -60,6 +60,7 @@ pub(super) struct Gate { } impl Gate { + #[cfg(CONFIG_KUNIT)] pub(super) fn new() -> Result> { Self::with_waiters(None) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5e6f20a3da43a0..da0123d8405dfc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -550,6 +550,8 @@ pub(crate) struct Queue { config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, last_published: u64, last_end: u64, released: bool, @@ -615,6 +617,7 @@ impl Queue { config_cache: None, ordinal: 0, submitted: 0, + claims: 0, last_published: 0, last_end: 0, released: false, @@ -2367,14 +2370,33 @@ impl Queue { Ok(self.reusable()) } + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + /// The queue's first command has been published: its identity and configuration exist. pub(crate) fn installed(&self) -> bool { self.installed } + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { - self.installed && self.owner.is_some() && self.retirement_ready + self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready && !self.growing && !self.quarantined && !self.awaiting_witness && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() && self.pool_phase == PoolPhase::Populated diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 88a3330155cac2..f1cefb99fe0055 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -299,6 +299,38 @@ impl Registry { Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) } + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + /// Binds `owner` to a reusable channel whose last owner released it, or /// names a vacant one to re-arm. None if no channel is reusable. fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { @@ -563,6 +595,16 @@ impl Registry { } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -616,6 +658,10 @@ pub(super) struct Backend { /// binding under the cell lock; a stale cell only sends the packet back to that path. #[pin] direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, } impl Backend { @@ -707,6 +753,7 @@ impl Backend { compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), }), GFP_KERNEL, )?, @@ -852,11 +899,14 @@ impl Backend { drop(queue); let mut state = self.shared.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation) + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) } fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - use kernel::time::{delay::fsleep, Delta}; admission.check()?; { let state = self.shared.state.lock(); @@ -874,6 +924,9 @@ impl Backend { let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); admission.check()?; @@ -899,9 +952,9 @@ impl Backend { match result { Ok(old) => { drop(old); return Ok(()); } Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } })(); @@ -948,6 +1001,8 @@ impl Backend { } loop { admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, objects, reservation) = { @@ -966,7 +1021,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -984,7 +1039,7 @@ impl Backend { // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission, false)?; + self.wait_channels(observed, admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -1018,7 +1073,7 @@ impl Backend { let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -1108,7 +1163,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -1237,6 +1292,10 @@ impl DeferredBatch { } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { use super::completion::Deferred; @@ -1570,12 +1629,15 @@ impl super::Firmware { )?; } let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.latch_observations(&mut registry.observations, masks)?; queue.retire( masks, @@ -1584,6 +1646,7 @@ impl super::Firmware { |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1606,11 +1669,22 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } @@ -1620,11 +1694,10 @@ impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1641,14 +1714,12 @@ impl Backend { result => return result, } } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted: the + // spared queue and its retained work stay intact past the deadline. + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - admission.check()?; - Err(EBUSY) } /// Steady-state publication into this owner's bound channel without the device mutex: only @@ -1706,6 +1777,98 @@ impl Backend { } } + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire()) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + /// Called with the device mutex held after a successful publication: remembers the bound /// channel for direct publication. fn remember_direct(&self, registry: &Registry) { @@ -1868,10 +2031,11 @@ impl Backend { } // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. let mut bound = true; let mut demand = None; - let mut recovery_polls = 0; - let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1880,11 +2044,13 @@ impl Backend { Ok(()) }; let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { admission.check()?; + let observed = progress.observe(); if !self.sources_published(packet)? { self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } // All DRM dependencies were satisfied before this scheduler callback. @@ -1897,15 +2063,11 @@ impl Backend { } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. - // Wait for one under the packet's admission deadline, as render - // publication does, instead of failing the packet. match packet.completion.work_state()?.acquire() { Err(error) if error == EBUSY => { drop(lease); - admission.check()?; self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); + self.wait_release(progress, observed, &admission)?; continue; } result => { @@ -1928,14 +2090,10 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; @@ -1955,28 +2113,18 @@ impl Backend { dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; - } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } if result.is_ok() { @@ -2103,6 +2251,8 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { + let mut freed = false; + let mut releasing = false; for mut queue in self .queues .compute @@ -2110,13 +2260,32 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.pool_release_stalled(now)) { return Err(ETIMEDOUT); @@ -2182,6 +2351,47 @@ impl Backend { } fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } + + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() + } + + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -2216,8 +2426,6 @@ impl Backend { Err(error) if error == EAGAIN => {} result => result?, } - admission.check()?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); admission.check() } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index b393afd9c8df29..39f628d007e186 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -55,6 +55,7 @@ impl Backend { drop(state); drop(binding); self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); self.shared.queue_events(); return Ok(()); } @@ -92,6 +93,7 @@ impl Backend { // A builder may own only an off-lock reservation when close arrives. // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -106,6 +108,8 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } @@ -212,8 +216,12 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute admission waits for its shared capacity here, as a fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => None, + } } fn publish(&self, packet: Arc) -> Result { @@ -241,6 +249,9 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -269,6 +280,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0a8ebc28c02e29..32b9acef94c0fd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -104,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index aaa2e281c69eef..16bd5828265a41 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -190,10 +190,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } From ccbcd078ab16a548806327187ab54cb1921cdf91 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:52:55 -0400 Subject: [PATCH 21/68] drm/asahi: wait for a G17 compute source only through its completion record A consumer of a firmware-visible compute source carried a completion-wait record before its descriptor and also named the source in its kick entry. The second dependency flagged the command as entry-signalled, so it also appended a predecessor record, and the firmware resolved the same wait twice. Keep only the completion-wait record, which orders the consumer behind its source by itself. Queue-to-queue hop 49.6 -> 46.0 us. Cross-queue order and data checks (10k iterations each), dependent reads under pressure, the compute/sync CTS gate and mixed churn pass. (cherry picked from commit 58ef65a530b707fb77878e9fdc68d577a27759fc) (cherry picked from commit 327e50fc45b57acf1954e8c2d78977198286feb0) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index f1cefb99fe0055..2c0d91820c9791 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1461,7 +1461,9 @@ impl super::Firmware { let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { return Err(EAGAIN); }; - merged.insert(sequence, dependency)?; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; if let Some(old) = native.iter_mut().find(|old| old.dependency.qid() == dependency.qid()) { From d7859c46857fecccd7c99d31936e112c23d807a0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:11:57 -0400 Subject: [PATCH 22/68] drm/asahi: add G17 command lifecycle tracepoints Add tracepoints for each command's publication (doorbell rung, with its out fence and the CPU counter), its retirement (with the firmware's GPU start and end timestamps), physical queue binds and event worker pass boundaries. With the counter and the GPU's system counter timestamps on the same events, a trace places firmware dispatch, GPU execution, firmware completion and every host stage of a command on one timeline. The events are disabled by default and cost a static-key test each. (cherry picked from commit 565dae906223948df97a252cc3274e8db6183363) (cherry picked from commit 6f4d3bbb47c1a7b19abef118b41b131440a8f414) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/Makefile | 3 +- drivers/gpu/drm/asahi/g17/mod.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 ++ drivers/gpu/drm/asahi/g17/queue/render.rs | 4 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 + drivers/gpu/drm/asahi/g17/trace.c | 38 +++++++ drivers/gpu/drm/asahi/g17/trace.h | 114 ++++++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 56 ++++++++++ 8 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/trace.c create mode 100644 drivers/gpu/drm/asahi/g17/trace.h create mode 100644 drivers/gpu/drm/asahi/g17/trace.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index d21dfc842c4454..aec066ef1d51e2 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/trace.o +CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o asahi-$(CONFIG_DRM_ASAHI_UAT_ADOPTION) += g15/uat_glue.o diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2f7cdf3ef81760..4332a73f4f4081 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -33,6 +33,7 @@ mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; mod sync_input; +mod trace; mod event; mod feed; mod fence; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index da0123d8405dfc..801e69c97ef797 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -589,6 +589,7 @@ impl Queue { let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, @@ -924,6 +925,10 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, + ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -1316,6 +1321,8 @@ impl Queue { } context.mark_published(); context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), + if same_vm { 2 } else { 1 }); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -2151,6 +2158,7 @@ impl Queue { } fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x10)?; + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index e063ef3a047fc7..faf31514a2f2f9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -954,6 +954,8 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, + packet); host.note_submission() })(); if let Err(error) = publish { @@ -1236,6 +1238,8 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, + &front.packet, [timestamps[0], timestamps[3]]); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e46c820efa1550..b1e8f82d858f99 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -323,8 +323,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -352,6 +354,7 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c new file mode 100644 index 00000000000000..feffb96315845b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#define CREATE_TRACE_POINTS +#include "trace.h" + +bool asahi_g17_trace_publish_enabled(void) +{ + return trace_asahi_g17_publish_enabled(); +} + +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence) +{ + trace_asahi_g17_publish(engine, qid, ordinal, ctx, seq, + fence ? fence->context : 0, fence ? fence->seqno : 0, + __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end) +{ + if (trace_asahi_g17_complete_enabled()) + trace_asahi_g17_complete(engine, qid, ordinal, ctx, seq, gpu_start, + gpu_end, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) +{ + trace_asahi_g17_bind(engine, qid, ctx, kind); +} + +void asahi_g17_trace_event(u8 phase) +{ + if (trace_asahi_g17_event_enabled()) + trace_asahi_g17_event(phase, __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h new file mode 100644 index 00000000000000..e1ef55715e312f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Command lifecycle tracepoints of the G17 runtime. Times are joined by + * (ctx, seq); `counter` is the CPU's virtual counter and `gpu_start`/`gpu_end` + * come from the firmware's completion record, so a reader can place firmware + * dispatch and completion on the host timeline. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM asahi_g17 + +#if !defined(ASAHI_G17_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define ASAHI_G17_TRACE_H + +#include + +TRACE_EVENT(asahi_g17_publish, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 fence_context, u64 fence_seqno, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, fence_context, fence_seqno, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->fence_context = fence_context; + __entry->fence_seqno = fence_seqno; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu fence=%llu:%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->fence_context, __entry->fence_seqno, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_complete, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, gpu_start, gpu_end, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, gpu_start) + __field(u64, gpu_end) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->gpu_start = gpu_start; + __entry->gpu_end = gpu_end; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu gpu=%llu..%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->gpu_start, __entry->gpu_end, __entry->counter) +); + +TRACE_EVENT(asahi_g17_bind, + TP_PROTO(u8 engine, u8 qid, u32 ctx, u8 kind), + TP_ARGS(engine, qid, ctx, kind), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ctx) + __field(u8, kind) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ctx = ctx; + __entry->kind = kind; + ), + TP_printk("engine=%u qid=%u ctx=%u kind=%u", + __entry->engine, __entry->qid, __entry->ctx, __entry->kind) +); + +TRACE_EVENT(asahi_g17_event, + TP_PROTO(u8 phase, u64 counter), + TP_ARGS(phase, counter), + TP_STRUCT__entry( + __field(u8, phase) + __field(u64, counter) + ), + TP_fast_assign( + __entry->phase = phase; + __entry->counter = counter; + ), + TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) +); + +#endif + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#include diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs new file mode 100644 index 00000000000000..a616b687352251 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a +//! static-key test in C while its event is disabled. + +use kernel::{bindings, dma_fence::RawDmaFence, prelude::*, sync::Arc}; + +unsafe extern "C" { + fn asahi_g17_trace_publish_enabled() -> bool; + fn asahi_g17_trace_publish(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + fence: *mut bindings::dma_fence); + fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + gpu_start: u64, gpu_end: u64); + fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_event(phase: u8); +} + +pub(super) const RENDER: u8 = 0; +pub(super) const COMPUTE: u8 = 1; + +/// Called once the command's doorbell has been rung. +pub(super) fn publish(engine: u8, qid: u8, ordinal: u32, packet: &Arc) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_publish_enabled() } { + return; + } + let fence = packet.completion.fence(); + // SAFETY: `fence` holds a reference for the duration of the call. + unsafe { + asahi_g17_trace_publish(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + fence.raw()) + }; +} + +/// Called when exact completion with GPU timestamps is retired. +pub(super) fn complete(engine: u8, qid: u8, ordinal: u32, packet: &super::job::Packet, + gpu: [u64; 2]) +{ + // SAFETY: Plain values only. + unsafe { + asahi_g17_trace_complete(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + gpu[0], gpu[1]) + }; +} + +/// A physical queue was bound to a logical owner: 0 fresh, 1 rebound, 2 same VM. +pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; +} + +/// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. +pub(super) fn event(phase: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_event(phase) }; +} From 0a64e52b2d35f0ccc5ec2c8d1ad40886acd8bb97 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:07:01 -0400 Subject: [PATCH 23/68] drm/asahi: ring the G17 compute pipe doorbell for every publication Compute rang its pipe and priority doorbell (pipe 2 in bits 1:0, class in bits 3:2) only for the first publication of a priority class, then the generic work doorbell for every command. Render rings its own pipe doorbell for every publication, and so does the M5 compute path. Ring the compute pipe doorbell for every publication instead of the generic one. gpubench on Neo (us/submit, on the integration branch): compute.light d2 60.3 -> 53.2, d64 22.8 -> 21.5, d1 163 -> 154..160, procs d16 unchanged (27.2), hop unchanged (48.7). Stress gate passes. (cherry picked from commit 398141cf95c44ac4d0073f3620a99d1235f31309) (cherry picked from commit e24e3bad9e7d3c6cf7079d683c9364d619cd3a55) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 801e69c97ef797..4189f687b0694d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2120,7 +2120,6 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); @@ -2151,13 +2150,12 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // Every publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); From d0e59c30171bbf35dc12398eea1f6f67f29568dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:38:41 -0400 Subject: [PATCH 24/68] drm/asahi: map the G17 compute queue graph cached for the firmware Render keeps its queue records in a firmware-cached prefix; the compute queue record, job-list head and item ring were mapped uncached. Allocate the compute graph from the retained object pool with its 32 KiB queue record, job-list head and item ring in a firmware-cached prefix, and move the producer/consumer words to their own uncached page after it, as native does. compute.light d64 22.2 -> 19.7 us/submit, 16-process compute 27.2 -> 25.7, queue-to-queue hop 47.5 -> 42.4 us. Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 488708add18c39f709d6a9b53736d339428cf5d0) (cherry picked from commit 94c66ecf24e6d931a434914d6a513ee36ff6ffd7) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 39 +++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4189f687b0694d..6ca63595154de9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -34,6 +34,10 @@ const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -120,7 +124,7 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, /// Announcements, which stay uncached for the firmware. @@ -157,11 +161,18 @@ impl Graph { support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), @@ -172,6 +183,10 @@ impl Graph { view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; @@ -259,9 +274,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -315,7 +330,7 @@ impl Graph { } fence(Ordering::SeqCst); self.graph - .word(POINTERS + self.pointers.producer)? + .word(CURSORS + self.pointers.producer)? .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); @@ -1689,11 +1704,11 @@ impl Queue { let producer = self .graph .graph - .word(POINTERS + self.graph.pointers.producer)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + target)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1764,7 +1779,7 @@ impl Graph { }; (object, slot * RECORD_STRIDE + offset) } else { - (&mut self.graph, 0x6800 + offset) + (&mut *self.graph, 0x6800 + offset) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1790,7 +1805,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -2262,7 +2277,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); From 660db571a4e2bf3a13157391e10513a8e09a73b5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:28:59 -0400 Subject: [PATCH 25/68] drm/asahi: wait for G17 render capacity on the release that frees it Render admission and publication still slept or polled: a 20 us..1 ms backoff when every work state of the context was taken, up to 200 x 1 ms while recovery was observed, an unconditional 1 ms sleep for render capacity (through the compute capacity wait), and a 1 ms condvar timeout as the wakeup for a pair on loan, firmware room or a pending teardown. Block on the event that ends each wait instead: - a render availability domain, signalled by every admission-pool return (lease drop, owner close, quarantine change, exit-kill hold release, processor stop), every pair return from a loan, and on each event pass while a render publisher waits, since firmware ring room, consumed releases and teardown steps have no event of their own; - a per-context work-state domain, signalled whenever a work state returns; - the preparation gate: a publisher that observed recovery waits for its closure (close() now wakes the gate's waiters) and then blocks in enter() until recovery reopens it. Render capacity first returns idle backing, then waits on the domain. Waiters sample a domain only after a failed check and retry once before blocking, so steady-state publication takes no extra lock. Deadlines only bound the waits; 10 ms slices only bound cancellation checks. Steady state is unchanged (render d1 135.7, d32 41.4, queues d8 62.6 us/submit); the mild stress gate passes. (cherry picked from commit a3fc054bc52c2bc82350e6073366045a1c3c9489) (cherry picked from commit db365ecc299f0663db8a629e70dfdb4d117eb95e) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/context.rs | 27 ++- drivers/gpu/drm/asahi/g17/mod.rs | 18 +- drivers/gpu/drm/asahi/g17/preparation.rs | 31 +++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 24 ++- drivers/gpu/drm/asahi/g17/runtime.rs | 16 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 + .../drm/asahi/g17/runtime/render/submit.rs | 192 +++++++++++++----- 7 files changed, 236 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 564d761d37f974..0d3948a2b69ffb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -122,6 +122,8 @@ pub(crate) struct Context { qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, work: Arc>, work_base: u64, render: Arc>>>, @@ -140,6 +142,7 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -199,6 +202,7 @@ impl Context { qos, owner_pid, states: WorkStates::new(), + state_returns, work, work_base, render, @@ -378,8 +382,9 @@ impl WorkStateLease { Err(EBUSY) }; }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + let initialized = self.context.work.lock().initialize(candidate); + if let Err(error) = initialized { + self.context.release_state(candidate); return Err(error); } match self @@ -388,11 +393,15 @@ impl WorkStateLease { { Ok(_) => Ok(candidate), Err(installed) => { - self.context.states.release(candidate); + self.context.release_state(candidate); Ok(installed) } } } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns + } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } @@ -433,7 +442,17 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word); + } + } +} + +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32) { + if self.states.release(word) { + self.state_returns.notify(); } } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4332a73f4f4081..c38c7e6e6bc931 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -64,7 +64,6 @@ use core::ops::Range; use core::sync::atomic::{ AtomicBool, AtomicU32, - AtomicU64, Ordering, // }; @@ -846,6 +845,11 @@ struct Shared { progress: Arc, /// Logical-queue creation waiting for a returned queue slot or execution context. queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, /// Unbound compute publishers which can reuse a retired channel. compute_demand: AtomicU32, #[pin] @@ -870,9 +874,6 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, - /// Counts `changed` notifications, so that a waiter which checked under - /// the device mutex can tell whether it already missed one. - changes: AtomicU64, /// The port of the session in `state`; null before boot and once detachment begins. port: core::sync::atomic::AtomicPtr, } @@ -966,13 +967,10 @@ impl Shared { /// Wakes the waiters for a returned pair, binding or capacity change. fn note_changed(&self) { - self.changes.fetch_add(1, Ordering::AcqRel); self.changed.notify_all(); + self.renders.notify(); } - fn changes(&self) -> u64 { - self.changes.load(Ordering::Acquire) - } /// The session's port, valid while `lease` is held: shutdown and device loss join every /// lease before they detach the session. @@ -1132,6 +1130,7 @@ impl Gpu { let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1165,6 +1164,7 @@ impl Gpu { channels, progress, queues, + renders, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1175,7 +1175,6 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), - changes: AtomicU64::new(0), port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, @@ -1327,6 +1326,7 @@ impl Gpu { shared.descriptor_flags, shared.variant, Some(shared.queues.clone()), + Some(shared.renders.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0df2c5ae2ebb25..3cafecf5d82067 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -179,8 +179,39 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ff2b294b6f35c6..424bb1b7601f5b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -379,15 +379,19 @@ pub(crate) struct Pool { max_queues: u32, /// Logical-queue creation waiting for a returned slot. waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { #[cfg(CONFIG_KUNIT)] pub(crate) fn new(max_queues: u32) -> Result> { - Self::with_waiters(max_queues, None) + Self::with_waiters(max_queues, None, None) } pub(crate) fn with_waiters( max_queues: u32, waiters: Option>, + renders: Option>, ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); @@ -399,10 +403,18 @@ impl Pool { queues: AtomicU32::new(0), max_queues, waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } /// Logical queues currently counted, including one being created. pub(crate) fn queues(&self) -> u32 { self.queues.load(Ordering::Acquire) @@ -579,7 +591,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -597,7 +609,7 @@ impl Pool { state.spared &= !released; state.kill_hold = 0; drop(state); - self.released.notify_all(); + self.returned(); } } @@ -715,7 +727,7 @@ impl RenderLease { } state.quarantined |= bit; drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. @@ -800,7 +812,7 @@ impl Drop for RenderLease { } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } @@ -839,7 +851,7 @@ impl Pool { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 2c0d91820c9791..1bd500bae6cb0a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -192,16 +192,18 @@ impl Registry { descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, ) -> impl Init { - Self::with_waiters(clusters, deflake, descriptor_flags, variant, None) + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) } - /// `queues` wakes logical-queue creation when a logical queue slot is returned. + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. pub(super) fn with_waiters( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { @@ -210,7 +212,7 @@ impl Registry { buffers <- BufferIds::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues)?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -735,7 +737,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -2346,12 +2348,6 @@ impl Registry { } impl Backend { - pub(super) fn wait_capacity(&self, started: u64) -> Result { - let check = || self.admission_open(); - let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission, true) - } - fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { self.reclaim_capacity(admission, reclaim_backing)?; kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index b1e8f82d858f99..1797ca5c0ce94f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -467,6 +467,11 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 663e62e8f86fca..bd83c81b1d593d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,11 +8,7 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the @@ -60,6 +56,9 @@ impl Backend { }; let started = super::super::now_ns(); let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; loop { self.admission_open()?; if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } @@ -97,7 +96,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); continue; @@ -131,7 +130,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); } @@ -140,36 +139,129 @@ impl Backend { } } - /// A pair is borrowed, still settling, or waiting for the release of the - /// scheduler it last ran for to be consumed and the pair parked. The event - /// worker observes both, so kick it on every poll; time out only after the - /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, seen: u64, sleep: bool) -> Result { + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + since: &mut Option, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { if !sleep { return Err(FALLBACK); } let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { - return Err(ETIMEDOUT); - } + let deadline = since.get_or_insert(now).saturating_add(super::super::WORK_RING_WAIT_NS); + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; self.shared.queue_events(); - // Every pair return notifies under the device mutex: wake on it rather - // than sleeping out a fixed interval behind a short observation loan. - let mut state = self.shared.state.lock(); - if self.shared.changes() == seen { - let _ = self.shared.changed.wait_timeout(&mut state, kernel::time::msecs_to_jiffies(1)); - } - Ok(()) + self.wait_domain(&self.shared.renders, sample, deadline, packet) } - /// A closing owner may still hold its pair until exact retirement. Wait - /// outside both the device mutex and preparation gate so the event worker - /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity(&self, observed: &mut Option, started: u64, sleep: bool) -> Result { if !sleep { return Err(FALLBACK); } - self.wait_capacity(started) + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = started + .saturating_add(super::super::WORK_RING_WAIT_NS) + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } } fn render_with_lease( @@ -193,13 +285,12 @@ impl Backend { let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; let mut busy_since = None; - let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); + let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); + let mut seen = None; + let mut states = None; loop { - let seen = self.shared.changes(); if packet.cancel_requested() { return Err(ECANCELED); } @@ -229,12 +320,15 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if state_polls == 1000 { - return Err(ETIMEDOUT); + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + Some(sample) => { + self.wait_domain(work.returns(), sample, deadline_ns, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { @@ -243,40 +337,42 @@ impl Backend { } match self.ensure_render(packet, slot, sleep) { Err(error) if error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, seen, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { if !sleep { return Err(FALLBACK); } - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared.preparations.wait_closure(gate, deadline_ns)?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, seen, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } From 1508e67cfa3b49ea4cc6b16f8772e4f1bc876615 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:34:06 -0400 Subject: [PATCH 26/68] drm/asahi: give the G17 render scheduler the pair's depth as credits A logical render queue owns exactly one physical pair, whose active ring holds RENDER_DEPTH (16) commands, but its scheduler had 32 credits. Jobs 17 to 32 entered run_job and parked there in the admission pool (or in the 1 ms busy wait) until a retirement freed a ring entry, holding the scheduler worker. With the credit limit equal to the pair depth the scheduler itself waits for the credit that the retirement returns, and run_job never blocks on a full pair. render.light d32 41.15 -> 41.26 us/submit (unchanged; the row is firmware bound), all other rows unchanged, mild stress gate passes. (cherry picked from commit 15de925f91cdb2a875e09fc1c6b64af79fa37ae2) (cherry picked from commit 06bc290c266abcb8b0d45f75c3369e231ae8c4a7) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/job.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 4c8546e60c217c..10ba27ff049215 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -14,7 +14,10 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -const RENDER_CREDITS: u32 = 32; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] From 2a5dc613b918b260108cf867a76e7177d7c61481 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:03:31 -0400 Subject: [PATCH 27/68] drm/asahi: keep a G17 render free list registered while the firmware is busy A render pool whose last render retired sent its final FreeListRelease right away: a control-ring record and a control doorbell for every render of a one-deep queue. The firmware processes each such message serially before it returns to the work rings, so with several queues active the releases stretched every completion burst (one release per render in render.light queues: about 44.6k per run). When another pair still has published work, leave a registration whose references drop to zero lingering instead; the owner's next render reuses it like an overlapping render, with no release and no new generation. A lingering registration ends, and its release is published, when its owner closes or moves to another pool (each event pass checks) or after the 16 ms firmware idle hold, for which the event worker schedules a timed pass. On an otherwise idle device the release is still sent at once: reusing a registration that went idle costs the firmware more than the release does (render d1 147 -> 184 us when every idle registration lingered). render.light queues d2/d4/d8: 80.8/67.3/62.6 -> 76.9/62.4/58.2 us/submit; render d1 148, d32 41.3 (unchanged). Releases per queues-d8 run: 44574 -> 18 (measured with every idle registration lingering). Mild stress gate passes. (cherry picked from commit 3d1f2047766c2d6e8d165a4af2cfe4adc0edd366) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/freelist.rs | 37 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/events.rs | 18 ++++++--- drivers/gpu/drm/asahi/g17/runtime/render.rs | 35 +++++++++++++++++- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 34 +++++++++++++++-- 4 files changed, 111 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3b31c3cead064a..9015dc82e40765 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -670,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -707,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -772,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 1797ca5c0ce94f..cdd8cc6d5dea55 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -399,6 +399,11 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; + let lingers = if firmware.recovery.pending() { + None + } else { + firmware.expire_render_lingers(now)? + }; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; let pools = firmware.service_compute_pools(deferred, now)?; @@ -441,11 +446,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 2877dea91d0ea7..d770c433528c9d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -609,7 +609,16 @@ impl crate::g17::queue::render::Host for Host<'_> { buffer_id: u8, ) -> Result { let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; if let Some(owners) = owners { self.firmware.port.qos_complete(&owners)?; @@ -645,6 +654,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 4713eb5b6b1522..78e5e3a219358b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases.iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() From 625cddd5187da1ae933b343d276d04ce2cad2e3e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:26:24 -0400 Subject: [PATCH 28/68] drm/asahi: take the G17 device mutex less often to publish a render A render publication into an owner's installed pair took the device mutex six times from the scheduler (seven from the ioctl): to fetch the admission pool (and, from the ioctl, again to try it), to check the pair's identities, to find the pair already built (constructing a throwaway set of build resources), to look for a sibling pair to warm, to lend the pair out and to publish it. The admission pool lives as long as the firmware session, so the logical queue keeps a reference to it. The identity check also reports whether the owner's pair is installed, idle and backed, in which case the first pass skips graph construction; every retry still goes through it. An owner has exactly one pair (kick.rs asserts it), so there is no sibling to warm. Steady state now takes the device mutex three times from either path. gpubench unchanged within noise (render d1 151.6, d32 42.8, queues d8 58.8 us/submit); the mild stress gate passes. (cherry picked from commit b7fa0d482eea670f9004422eaf57703a7ec8d4e5) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 6 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 4 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 22 ++++++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 4 ++ .../drm/asahi/g17/runtime/render/submit.rs | 41 +++++++++---------- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 424bb1b7601f5b..b3cd061e0e912c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -686,6 +686,12 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 1bd500bae6cb0a..30816cd1cbdb6f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -664,6 +664,8 @@ pub(super) struct Backend { /// channel from being shared away between prepare_job and run_job. #[pin] claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, } impl Backend { @@ -745,6 +747,7 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( Arc::pin_init( pin_init!(Self { @@ -756,6 +759,7 @@ impl Backend { closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, }), GFP_KERNEL, )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index d770c433528c9d..6360317b278513 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -326,6 +326,28 @@ impl Registry { Ok(None) } + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 04afe2e05c1079..62ca7895a86f8e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -593,6 +593,10 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index bd83c81b1d593d..9d77a1e104328e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -44,16 +44,15 @@ impl Backend { /// Reserve identities before attaching a lease to the packet. An empty /// logical slot must not hide reusable physical channels when the shared /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. fn acquire_render_with_ids( &self, packet: &Arc, mut next: Option, sleep: bool, - ) -> Result { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; let started = super::super::now_ns(); let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); // A capacity refusal samples the render domain and retries once; only @@ -108,7 +107,10 @@ impl Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => return Ok(lease), + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; if reclaimed.is_some() { @@ -282,7 +284,7 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early, sleep)?; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut busy_since = None; @@ -335,7 +337,14 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot, sleep) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { Err(error) if error == ENOSPC => { self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; @@ -386,19 +395,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); From 1b16ae146e7521c648a29c30ce10833c47be849c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:43:52 -0400 Subject: [PATCH 29/68] drm/asahi: publish a G17 render into its pair without the device mutex Every render publication held the device mutex, which the event worker also needs for each pass, across the whole publication: dependency and hazard collection, kick and descriptor commits, QoS accounting, both work-ring records and producers, and the doorbell. In the steady state (the owner's graph is installed and registered at its priority, last ran this owner's scheduler, needs no growth, has nothing to replay or settle, and the command has no compute prefix) the device mutex now only lends the pair out, together with its free-list preparation and a parameter-buffer reference, and takes it back. Preparation and publication in between take the port: a render publication lock that every render publication holds, with or without the device mutex (the tiling and fragment work rings keep one host producer at a time, and attachment overlaps keep their publication order), QoS accounting, the completion stamps, the doorbell and the full-effort witness. The port gains views of the render work rings and of the stamp page. The free-list lease leaves preparation, or is rolled back with the parameter-buffer reference, when the pair returns; while it is lent the event worker skips the pair, so no retirement can touch its commands or references. A recovery that starts during the publication fails it with EIO, as on the compute path. Anything else, including a first publication, growth and owner changes, still publishes under the device mutex. During gpubench render rows 306k of 320k publications took the new path. render.light d2/d8/d32 68.9/45.2/42.8 -> 65.9/43.5/41.1 us/submit, d1 150.2 (151.6), queues d8 58.7 (58.8); the rows stay firmware bound. Mild stress gate passes. (cherry picked from commit d272a7060d20a0beb635ca8426c8822b8e823a43) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/initdata.rs | 25 +- drivers/gpu/drm/asahi/g17/port.rs | 125 +++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 49 +++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/direct.rs | 366 ++++++++++++++++++ .../drm/asahi/g17/runtime/render/submit.rs | 14 +- 6 files changed, 566 insertions(+), 14 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/direct.rs diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6d0b2f9951956e..8f7551b1763c9e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -659,15 +659,16 @@ impl InitData { Ok(()) } - /// Raw views of the memory publishers share without the device mutex: every compute work - /// ring, the QoS table and the recovery state word. They stay valid for this session. + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. pub(super) fn port_views( &self, mailbox: *const super::Mailbox, ) -> Result { use super::fw::{channels, queue::{DataMaster, Policy}}; - let ring = |priority: u8| -> Result { - let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let word = |offset: usize| -> Result<*const AtomicU32> { @@ -686,12 +687,18 @@ impl InitData { producer: word(abi::ChannelState::PRODUCER)?, }) }; - let mut compute = [ring(0)?; Policy::PRIORITIES as usize]; - for (priority, view) in compute.iter_mut().enumerate().skip(1) { - *view = ring(priority as u8)?; - } + let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; Ok(super::port::Views { - compute, + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, qos: self .qos()? .pointer(0, core::mem::size_of::())?, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 63eab753632be7..1adf9f96904565 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -7,7 +7,9 @@ //! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every //! lease before they detach the session. Each piece below is atomic or has its own lock. //! -//! Lock order: device mutex -> compute queue cell -> the locks of this port. +//! Lock order: device mutex -> compute queue cell -> the locks of this port. The render +//! publication lock comes first among the port's locks: device mutex -> render publication +//! lock -> admission pool, QoS, mailbox. Its holder never takes the device mutex. use super::{ fw::{ @@ -18,9 +20,9 @@ use super::{ }; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; use kernel::{ - new_spinlock, + new_mutex, new_spinlock, prelude::*, - sync::{Arc, SetOnce, SpinLock}, + sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, }; const PRIORITIES: usize = Policy::PRIORITIES as usize; @@ -62,6 +64,10 @@ impl RingView { /// Raw views of the session's firmware-shared memory, fixed once the firmware is built. pub(super) struct Views { pub(super) compute: [RingView; PRIORITIES], + pub(super) tiling: [RingView; PRIORITIES], + pub(super) fragment: [RingView; PRIORITIES], + /// Completion stamp page A: one word per QID, written by the firmware. + pub(super) stamps: *const AtomicU32, pub(super) qos: *mut u8, pub(super) recovery: *const AtomicU32, pub(super) mailbox: *const super::Mailbox, @@ -81,6 +87,11 @@ pub(super) struct Port { rings: SpinLock<[u32; PRIORITIES]>, #[pin] qos: SpinLock, + /// Serializes every render publication, with or without the device mutex: the tiling + /// and fragment work rings have one host producer at a time, and same-VM attachment + /// overlaps are ordered by publication order (see hazard.rs). + #[pin] + render: Mutex<()>, /// The accepted recovery generation, mirrored for publishers without the device mutex. generation: AtomicU64, /// Device publication order shared by compute and render commands. @@ -114,6 +125,7 @@ impl Port { views: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), @@ -196,6 +208,113 @@ impl Port { Ok(next) } + /// Taken around every render publication; see `render`. + pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { + self.render.lock() + } + + fn render_rings(&self, priority: u8) -> Result<(RingView, RingView)> { + let views = self.views()?; + let index = usize::from(priority); + Ok(( + *views.fragment.get(index).ok_or(EINVAL)?, + *views.tiling.get(index).ok_or(EINVAL)?, + )) + } + + /// Both render work rings of `priority` have a free slot. Only firmware consumption, + /// never a publisher without the render lock, changes this. + pub(super) fn render_room(&self, _guard: &MutexGuard<'_, ()>, priority: u8) -> Result { + let (fragment, tiling) = self.render_rings(priority)?; + Ok(fragment.free()? > 0 && tiling.free()? > 0) + } + + /// Copies a render pair's fragment, then tiling record into the slots after each + /// producer without exposing them, as `InitData::publish_work_pair` does for a deferred + /// pair. Returns the producers that expose them. + pub(super) fn write_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + fragment: &WorkSlot, + tiling: &WorkSlot, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let (fragment_ring, tiling_ring) = self.render_rings(priority)?; + let slot = |ring: &RingView| -> Result<(*mut u8, u32)> { + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EINVAL); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. + Ok((unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }, next)) + }; + let (fragment_slot, fragment_next) = slot(&fragment_ring)?; + let (tiling_slot, tiling_next) = slot(&tiling_ring)?; + let copy = |record: &WorkSlot, destination: *mut u8| { + // SAFETY: The slot after the producer is host-owned until the producer moves past + // it, and the render lock excludes every other host writer of this ring. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + destination, + core::mem::size_of::(), + ) + } + }; + copy(fragment, fragment_slot); + commit_accounting()?; + fence(Ordering::SeqCst); + publish_late_tiling(); + copy(tiling, tiling_slot); + fence(Ordering::SeqCst); + Ok([fragment_next, tiling_next]) + } + + /// Exposes a pair written by `write_render_pair`, fragment first, as + /// `InitData::publish_retained_work_pair` does: the producers must still precede `next`. + pub(super) fn expose_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + next: [u32; 2], + ) -> Result { + let (fragment_ring, tiling_ring) = self.render_rings(priority)?; + for (ring, next) in [(&fragment_ring, next[0]), (&tiling_ring, next[1])] { + let [_, _, producer] = ring.cursors()?; + if next != (producer + 1) % WORK_RING_SLOTS { + return Err(EBUSY); + } + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*fragment_ring.producer).store(next[0], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + // SAFETY: As above. + unsafe { (*tiling_ring.producer).store(next[1], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The completion stamp of `qid` in page A. + pub(super) fn stamp(&self, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let base = self.views()?.stamps; + // SAFETY: Page A holds one word per QID up to QID_MAX; the session retains it. + let value = unsafe { (*base.add(usize::from(qid))).load(Ordering::Relaxed) }; + fence(Ordering::Acquire); + Ok(value) + } + fn qos_view(&self) -> Result> { // SAFETY: The session retains the QoS table for at least as long as this port. unsafe { qos::View::from_raw(self.views()?.qos) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index faf31514a2f2f9..3e65ea3b9a367f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1787,6 +1787,55 @@ impl Pair { } /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier)?; + for dependency in frontier.as_slice() { + *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 6360317b278513..aa437b8a21be2c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs new file mode 100644 index 00000000000000..00e0037ebae608 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Steady-state render publication without the device mutex. +//! +//! The device mutex lends the owner's installed pair out, together with its +//! free-list preparation and a parameter-buffer reference, and takes it back. +//! Between the two, command preparation and publication take only the port: +//! its render publication lock (the render work rings' single producer and +//! the publication order of attachment hazards), QoS accounting, the stamps +//! and the doorbell. While the pair is lent, the event worker skips it, so no +//! retirement can complete its commands or their free-list references. + +use super::super::{Backend, DeferredBatch}; +use crate::g17::{ + job::Packet, + object::Allocator, + port::Port, + queue::render::{Pair, PublicationLease, RenderAddresses}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::{Arc, MutexGuard}, +}; + +/// A pair lent out for one publication, with what that publication holds. +pub(in crate::g17::runtime) struct Loan { + pair: Option>, + preparation: Option, + buffer: u8, + lease: PublicationLease, + addresses: RenderAddresses, + uat: Arc, +} + +/// Publication host of a lent pair. Registration, first-publication and +/// growth paths never run here: the loan requires an installed, registered +/// graph that needs no growth. +struct DirectHost<'a> { + port: &'a Port, + render: &'a MutexGuard<'a, ()>, + generation: u64, + activation: Option<(u16, u64)>, +} + +impl crate::g17::queue::render::Host for DirectHost<'_> { + fn next_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) + } + fn qos_publish( + &mut self, + owner: crate::g17::qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + Err(EAGAIN) + } + fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { + // An installed graph published its identities with its first command. + Ok(()) + } + fn publish_outer_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + self.port + .write_render_pair(self.render, priority, fragment, tiling, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + // The lease leaves preparation when the pair is returned; nothing can + // observe it in between because the pair is lent. + self.activation = Some((id, generation)); + Ok(()) + } + fn publish_retained_pair( + &mut self, + priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + self.port.expose_render_pair(self.render, priority, next) + } + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { + self.port + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2) + } + fn note_submission(&mut self) -> Result { + // Counted, with full effort witnessed, before the pair was lent. + Ok(()) + } + fn complete_render( + &mut self, + _owners: Option<[crate::g17::qos::Owner; 2]>, + _pool_id: u16, + _generation: u64, + _buffer_id: u8, + ) -> Result { + Err(EINVAL) + } +} + +impl crate::g17::Firmware { + /// Lends `owner`'s pair in `slot` for a publication of `packet`, or None if + /// the pair is not in the steady state the direct path requires. + fn lend_render( + &mut self, + slot: u8, + owner: u64, + packet: &Arc, + priority: u8, + generation: u64, + ) -> Result> { + if self.recovery.pending() || self.recovery.generation() != generation { + return Ok(None); + } + let Ok(entry) = self.queues.render.entry_mut(slot) else { + return Ok(None); + }; + if !entry.published + || entry.closed + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Ok(None); + } + let Some(pair) = entry.pair.as_mut() else { + return Ok(None); + }; + if !pair.matches(owner, &packet.context)? + || !pair.direct_ready(&packet.context, priority) + || pair.pool().vacant() + || !matches!(pair.host_growth_target(packet), Ok(None)) + { + return Ok(None); + } + let qids = pair.qids(); + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let (buffer, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer)?; + return if error == EAGAIN || error == EBUSY { + Ok(None) + } else { + Err(error) + }; + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let pair = match self.queues.render.take_pair(slot, owner, packet) { + Ok(pair) => pair, + Err(error) => { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer)?; + return if error == EBUSY { Ok(None) } else { Err(error) }; + } + }; + // The selected scheduler is retained through release acknowledgement, + // as on the device-mutex path. + packet.context.mark_published(); + Ok(Some(Loan { + pair: Some(pair), + preparation: Some(preparation), + buffer, + lease, + addresses, + uat: self.uat.clone(), + })) + } + + /// Takes a lent pair back and settles what its publication held: the + /// free-list preparation is activated once published, rolled back with the + /// parameter-buffer reference otherwise. + fn return_render( + &mut self, + slot: u8, + loan: &mut Loan, + activation: Option<(u16, u64)>, + packet: &Arc, + ) -> Result { + self.queues.render.return_pair(slot, &mut loan.pair)?; + if self.queues.render.take_missed_event(slot) { + self.primary.state.shared.queue_events(); + } + let entry = self.queues.render.entry_mut(slot)?; + // A failure or timeout recorded against the lent pair is settled by the + // event worker's collector, as after any other loan. + if entry.pending_error.is_some() || entry.pending_timeout { + self.primary.state.shared.queue_events(); + } + let pair = entry.pair.as_mut().ok_or(EIO)?; + if entry.closed { + pair.release_owner(); + } + if !packet.is_published() { + pair.discard_prepared(packet)?; + } + if let Some((id, generation)) = activation { + if self.queues.render.pools.activate(id, generation).is_err() { + self.primary.state.shared.lose_device_quietly(); + } + } + let preparation = loan.preparation.take().ok_or(EIO)?; + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(loan.buffer)?; + } + Ok(()) + } +} + +impl Backend { + /// Publishes `packet` into the owner's installed pair with the device + /// mutex held only to lend the pair out and take it back. None leaves the + /// packet unpublished for the device-mutex path. + pub(in crate::g17::runtime) fn render_direct( + &self, + packet: &Arc, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Option { + // A compute prefix needs the compute queue's state; leave it to the + // device-mutex path. + if packet.is_published() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || packet.order.wait_through[1].unwrap_or(0) != 0 + || self.admission_open().is_err() + { + return None; + } + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let priority = packet.context.policy().priority(); + let generation = port.generation(); + let dev = self.shared.drm().ok()?; + let mut loan = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut()?; + if !lease.is_current() { + return None; + } + match firmware.lend_render(slot, self.owner, packet, priority, generation) { + Ok(Some(loan)) => loan, + Ok(None) => return None, + Err(error) => return Some(Err(error)), + } + }; + let mut activation = None; + let result = (|| { + let pair = loan.pair.as_mut().ok_or(EIO)?; + let alloc = Allocator { + dev: &dev, + uat: &loan.uat, + }; + pair.prepare(&alloc, packet, loan.addresses)?; + let render = port.render_lock(); + if !lease.is_current() || !port.render_room(&render, priority)? { + return Err(EBUSY); + } + let mut dependencies = pair.own_dependencies(self.owner, packet)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.admission.publication_barriers( + self.owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } + let mut host = DirectHost { + port, + render: &render, + generation, + activation: None, + }; + let result = pair.publish( + packet.clone(), + &dependencies, + loan.lease, + &mut host, + &mut |event| deferred.push(event), + ); + activation = host.activation; + result?; + // A recovery that started meanwhile may have discarded the commands. + if port.generation() != generation || port.recovery_state()? != 0 { + return Err(EIO); + } + Ok(()) + })(); + let returned = { + let mut state = self.shared.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => { + let returned = firmware.return_render(slot, &mut loan, activation, packet); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); + } + returned + } + // Shutdown joins every preparation lease before detaching the + // session, so this cannot happen while `lease` is held. + None => Err(ENODEV), + } + }; + self.shared.note_changed(); + drop(lease); + if let Err(error) = returned { + self.shared.lose_device_quietly(); + return Some(Err(error)); + } + match result { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 9d77a1e104328e..a9dac3dab2f73c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -287,6 +287,11 @@ impl Backend { let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } let mut busy_since = None; let capacity_started = super::super::now_ns(); let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); @@ -549,15 +554,20 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; if let Some(footprint) = packet.footprint.as_ref() { dependencies.hazard_count = self.queues.admission.publication_barriers( owner, From b64d0fa8bf570202296df46607ebf3946336273d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:52:57 -0400 Subject: [PATCH 30/68] drm/asahi: lend a G17 render pair without the identity check first After the direct render path, a steady-state publication still took the device mutex three times: once to check the pair's identities and readiness, then to lend the pair and to take it back. The lend itself validates everything the identity check does. The logical queue remembers the pair it last published to directly. A lease on that pair (the ioctl's, or a non-blocking admission from the scheduler) goes straight to the lend; anything else, and any lend that declines, takes the full path with the identity check. gpubench unchanged within noise (render d1 150.8, d32 42.9, queues d8 58.4 us/submit); the mild stress gate passes. (cherry picked from commit 6428f7fd42fe075981984da569b0fa076f59a127) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +++ .../drm/asahi/g17/runtime/render/direct.rs | 7 +++++- .../drm/asahi/g17/runtime/render/submit.rs | 25 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 30816cd1cbdb6f..b07423bd22115d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -666,6 +666,9 @@ pub(super) struct Backend { claimed: kernel::sync::Mutex>>, /// Render pair admission, shared by every logical queue of the session. admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -760,6 +763,7 @@ impl Backend { direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), }), GFP_KERNEL, )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index 00e0037ebae608..cd7bad6ecce5d6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -360,7 +360,12 @@ impl Backend { } match result { Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, - result => Some(result), + result => { + if result.is_ok() { + self.direct_slot.store(u32::from(slot), Ordering::Relaxed); + } + Some(result) + } } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index a9dac3dab2f73c..733a570f0e8610 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -284,6 +284,31 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() + } + lease => lease, + }; let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; From f720d9af3581a147ac747571569762282c7bad2c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 01:04:35 -0400 Subject: [PATCH 31/68] drm/asahi: wait for G17 render admission before run_job The scheduler's prepare step returned no fence for render jobs, so a job that could not get a work state or a pair admission (a cross-VM attachment overlap, a crowded VM, no pair capacity) waited for it inside run_job, holding the scheduler worker. prepare_job now checks both without taking anything: a work state of the context, and whether the admission pool would admit the command (its owner's pair has room and no overlap plan says wait, or a free pair exists for an owner without one). When either is missing it returns a fence of the domain whose return provides it: the context's work-state domain, or the render domain that every admission-pool return signals. The scheduler calls prepare again once it fires. Each check runs once more after sampling its domain, so the common case takes no domain lock. gpubench unchanged within noise (render d1 151.8, d32 42.7, queues d8 58.8 us/submit); the mild stress gate passes. (cherry picked from commit 8cf02f589bce34b13002cbdf0f3b0361c4a8e66b) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 40 +++++++++++++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 +- .../drm/asahi/g17/runtime/render/submit.rs | 49 +++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index b3cd061e0e912c..e1e7552c243f24 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -298,6 +298,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -555,6 +577,24 @@ impl Pool { record, })) } + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } pub(crate) fn warmable_sibling( self: &Arc, owner: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 39f628d007e186..d19f51c08dd290 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -216,11 +216,12 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - /// Compute admission waits for its shared capacity here, as a fence. + /// Compute and render admission wait for their shared capacity here, as a + /// fence. fn prepare(&self, packet: &Arc) -> Option { match packet.engine() { Engine::Compute => self.prepare_compute(packet), - Engine::Render => None, + Engine::Render => self.admit_render(packet), } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 733a570f0e8610..1ef6742b689a78 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -141,6 +141,55 @@ impl Backend { } } + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire() { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire() { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + /// Conditions which end a publication that has not become visible yet. fn publication_open(&self, packet: &Packet) -> Result { self.admission_open()?; From a20f04d4ffba40e8d711ed36b238168323e04395 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:16:03 -0400 Subject: [PATCH 32/68] drm/asahi: trace the G17 core power state at each work doorbell Add an asahi_g17_doorbell tracepoint that samples the core power register just before a compute or render work doorbell rings. The register is read only while the event is enabled, so tracing can tell whether a slow doorbell-to-start interval coincides with powered-down cores. (cherry picked from commit 550fb355e9a30c0db9c59fe35a931f1db9c6b5c4) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/port.rs | 10 ++++++++-- drivers/gpu/drm/asahi/g17/runtime/render.rs | 3 +++ drivers/gpu/drm/asahi/g17/trace.c | 10 ++++++++++ drivers/gpu/drm/asahi/g17/trace.h | 17 +++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 13 +++++++++++++ drivers/gpu/drm/asahi/regs.rs | 5 +++++ 7 files changed, 57 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c38c7e6e6bc931..33cc7cede6c1a1 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1363,7 +1363,7 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; } let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach(session.init.port_views(mailbox)?)?; + session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 1adf9f96904565..146b83972f2033 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -82,6 +82,8 @@ unsafe impl Sync for Views {} #[pin_data] pub(super) struct Port { views: SetOnce, + /// Register access for tracing the core power state at the doorbell. + regs: SetOnce, /// Compute work-ring slots reserved by publishers that have not exposed them yet. #[pin] rings: SpinLock<[u32; PRIORITIES]>, @@ -123,6 +125,7 @@ impl Port { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), + regs: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), render <- new_mutex!((), "g17::Port::render"), @@ -136,8 +139,8 @@ impl Port { } /// Called once while the session is built, before any publisher can see the port. - pub(super) fn attach(&self, views: Views) -> Result { - if self.views.populate(views) { + pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { + if self.views.populate(views) && self.regs.populate(regs) { Ok(()) } else { Err(EBUSY) @@ -344,6 +347,9 @@ impl Port { pub(super) fn notify(&self, message: u64) -> Result { // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. let mailbox = unsafe { &*self.views()?.mailbox }; + super::trace::doorbell(super::trace::COMPUTE, || { + self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) + }); mailbox.send(super::EP_DOORBELL, message) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aa437b8a21be2c..f4f45a98005c33 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -612,6 +612,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index feffb96315845b..fc189458af32df 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -31,6 +31,16 @@ void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) trace_asahi_g17_bind(engine, qid, ctx, kind); } +bool asahi_g17_trace_doorbell_enabled(void) +{ + return trace_asahi_g17_doorbell_enabled(); +} + +void asahi_g17_trace_doorbell(u8 engine, u32 power) +{ + trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); +} + void asahi_g17_trace_event(u8 phase) { if (trace_asahi_g17_event_enabled()) diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index e1ef55715e312f..e00271249ce647 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -91,6 +91,23 @@ TRACE_EVENT(asahi_g17_bind, __entry->engine, __entry->qid, __entry->ctx, __entry->kind) ); +TRACE_EVENT(asahi_g17_doorbell, + TP_PROTO(u8 engine, u32 power, u64 counter), + TP_ARGS(engine, power, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u32, power) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->power = power; + __entry->counter = counter; + ), + TP_printk("engine=%u power=%#x counter=%llu", __entry->engine, __entry->power, + __entry->counter) +); + TRACE_EVENT(asahi_g17_event, TP_PROTO(u8 phase, u64 counter), TP_ARGS(phase, counter), diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index a616b687352251..d0150c73189581 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -12,6 +12,8 @@ unsafe extern "C" { fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, gpu_start: u64, gpu_end: u64); fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_doorbell_enabled() -> bool; + fn asahi_g17_trace_doorbell(engine: u8, power: u32); fn asahi_g17_trace_event(phase: u8); } @@ -49,6 +51,17 @@ pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; } +/// A work doorbell is about to ring; `power` samples the core power state +/// only while the event is enabled. +pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_doorbell_enabled() } { + return; + } + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_doorbell(engine, power()) }; +} + /// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. pub(super) fn event(phase: u8) { // SAFETY: Plain values only. diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 0a4ee8e7563ec2..6a7bd5057fa543 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -707,6 +707,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { From f09255283d074c36d8ee518b90b654563bb7e92c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:32:18 -0400 Subject: [PATCH 33/68] drm/asahi: trace G17 host-to-firmware messages Add an asahi_g17_message tracepoint for every mailbox message, device-control record (by opcode) and effort update the host sends to the firmware, so a trace can show what the firmware was handling when a doorbell was slow to start. (cherry picked from commit f381ac9b71ebcb70eb72497ec4115553856a31fc) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/channel.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 2 ++ drivers/gpu/drm/asahi/g17/port.rs | 1 + drivers/gpu/drm/asahi/g17/trace.c | 6 ++++++ drivers/gpu/drm/asahi/g17/trace.h | 17 +++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 12 ++++++++++++ 6 files changed, 39 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a1661..d96924cbbf9f91 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -138,6 +138,7 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 33cc7cede6c1a1..0bfc0ba05ccd3e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -550,6 +550,7 @@ impl Coprocessor { /// Sends a host notification to the firmware. fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); self.rtkit.send(EP_DOORBELL, msg) } @@ -708,6 +709,7 @@ impl Firmware { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 146b83972f2033..d8f7f4ddff8a12 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -350,6 +350,7 @@ impl Port { super::trace::doorbell(super::trace::COMPUTE, || { self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) }); + super::trace::message(super::trace::MAILBOX, message); mailbox.send(super::EP_DOORBELL, message) } diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index fc189458af32df..ae977c13aab3bd 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -41,6 +41,12 @@ void asahi_g17_trace_doorbell(u8 engine, u32 power) trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); } +void asahi_g17_trace_message(u8 kind, u64 value) +{ + if (trace_asahi_g17_message_enabled()) + trace_asahi_g17_message(kind, value, __arch_counter_get_cntvct()); +} + void asahi_g17_trace_event(u8 phase) { if (trace_asahi_g17_event_enabled()) diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index e00271249ce647..7e5a50e774b598 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -108,6 +108,23 @@ TRACE_EVENT(asahi_g17_doorbell, __entry->counter) ); +TRACE_EVENT(asahi_g17_message, + TP_PROTO(u8 kind, u64 value, u64 counter), + TP_ARGS(kind, value, counter), + TP_STRUCT__entry( + __field(u8, kind) + __field(u64, value) + __field(u64, counter) + ), + TP_fast_assign( + __entry->kind = kind; + __entry->value = value; + __entry->counter = counter; + ), + TP_printk("kind=%u value=%#llx counter=%llu", __entry->kind, __entry->value, + __entry->counter) +); + TRACE_EVENT(asahi_g17_event, TP_PROTO(u8 phase, u64 counter), TP_ARGS(phase, counter), diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index d0150c73189581..57db5abaccef86 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -12,6 +12,7 @@ unsafe extern "C" { fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, gpu_start: u64, gpu_end: u64); fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_message(kind: u8, value: u64); fn asahi_g17_trace_doorbell_enabled() -> bool; fn asahi_g17_trace_doorbell(engine: u8, power: u32); fn asahi_g17_trace_event(phase: u8); @@ -62,6 +63,17 @@ pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { unsafe { asahi_g17_trace_doorbell(engine, power()) }; } +pub(super) const MAILBOX: u8 = 0; +pub(super) const CONTROL: u8 = 1; +pub(super) const EFFORT: u8 = 2; + +/// A host-to-firmware message: a mailbox message, a device-control record by +/// opcode, or a new performance effort value. +pub(super) fn message(kind: u8, value: u64) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_message(kind, value) }; +} + /// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. pub(super) fn event(phase: u8) { // SAFETY: Plain values only. From aefc76a85320c2f5038b1a23b8a71b65bc73b9bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 02:09:59 -0400 Subject: [PATCH 34/68] drm/asahi: maintain cumulative completion scopes without repeated fence merges Publish queue-scoped completion outputs with bounded callback bookkeeping while preserving independent GPU execution and native input dependencies. Signal ready scopes iteratively, propagate errors and deadline hints, and release completed history. Validated on Neo with the full benchmark and stress gate, published software-fence ordering, 256 queued completions, deadline forwarding and injected errors. Same-boot compute submission CPU drops from about 14 us to 10 us; GPU throughput remains near 20 us. (cherry picked from commit adcd56e020362664c7bc3af288e1ad812df7b4c7) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/file.rs | 45 +++- drivers/gpu/drm/asahi/g17/fence.rs | 42 ++- drivers/gpu/drm/asahi/g17/mod.rs | 4 +- drivers/gpu/drm/asahi/g17/sync_input.c | 72 ++++-- drivers/gpu/drm/asahi/g17/sync_output.c | 138 ++++++++++ drivers/gpu/drm/asahi/g17/sync_output.h | 23 ++ drivers/gpu/drm/asahi/g17/sync_output.rs | 69 +++++ drivers/gpu/drm/asahi/g17/sync_scope.c | 315 +++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/sync_scope.h | 15 ++ include/uapi/drm/asahi_drm.h | 17 ++ rust/kernel/drm/syncobj.rs | 8 + 12 files changed, 719 insertions(+), 31 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.h create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.rs create mode 100644 drivers/gpu/drm/asahi/g17/sync_scope.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_scope.h diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index aec066ef1d51e2..037ffb71a9d4cf 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/trace.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index c627bd9c6b8a05..0c4c6048408262 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -127,10 +127,18 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, + bundled: bool) -> Result + { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -151,6 +159,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -173,12 +183,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -193,12 +201,14 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -214,7 +224,25 @@ impl SyncItem { // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push(Self::parse_one(file, sync, i >= in_count, + allow_cumulative)?, GFP_KERNEL)?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], anchor - in_count as usize)?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -1398,7 +1426,8 @@ impl File { copy_commands(&mut vec)?; } let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, + gpu.as_any().is::())?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index da883cede52df7..8245e8bca71e65 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -19,6 +19,16 @@ use kernel::{ pub(crate) struct CompletionFence; +unsafe extern "C" { + fn asahi_g17_signal_fence(fence: *mut bindings::dma_fence); +} + +fn signal_exported(fence: &dyn RawDmaFence) { + // SAFETY: The borrowed fence remains owned throughout synchronous signaling. + // Ready completion scopes are drained iteratively after its lock is released. + unsafe { asahi_g17_signal_fence(fence.raw()) }; +} + #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -52,6 +62,7 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { /// Submission never owns this fence, so exported references cannot form a cycle. pub(crate) struct ExportFence { source: Option>, + scoped: bool, } static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); @@ -65,7 +76,7 @@ impl FenceOps for ExportFence { #[inline(never)] fn export_fallback(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, ExportFence { source: None })?; + let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; // SAFETY: Unique, initialized and still private. Only this constructor // publishes ExportFence objects; its payload is immutable after sharing. let ops = unsafe { @@ -179,7 +190,7 @@ impl Submission { let error = self.error.load(Ordering::Acquire); if error != 0 { fence.set_error(Error::from_errno(error)); } else { self.release_state(); } - fence.signal(); + if fence.scoped { signal_exported(fence); } else { fence.signal(); } } } } @@ -332,30 +343,42 @@ pub(crate) struct Outputs { input_count: usize, submission: Option, result: Result, + cumulative: Option, } impl Outputs { pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, input_count: usize, allocation_failure: &Fence) -> Result { if input_count > syncs.len() { return Err(EINVAL); } + let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - install(syncs.drain(input_count..), allocation_failure); + if let Some(bundle) = &mut cumulative { + bundle.fail(allocation_failure); + } else { + install(syncs.drain(input_count..), allocation_failure); + } return Err(error); } }; Ok(Self { fallback: Some(fallback), syncs, input_count, - submission: None, result: Err(ECANCELED) }) + submission: None, result: Err(ECANCELED), cumulative }) } pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { let source = Submission::new(context, firmware_source)?; let mut fence = self.fallback.take().ok_or(EINVAL)?; fence.source = Some(source); + fence.scoped = self.cumulative.is_some(); let submission = Aggregate(fence.into()); - install(self.syncs.drain(self.input_count..), &submission.fence()); self.submission = Some(submission.clone()); + if let Some(bundle) = &mut self.cumulative { + bundle.publish(&submission.fence())?; + self.syncs.truncate(self.input_count); + } else { + install(self.syncs.drain(self.input_count..), &submission.fence()); + } Ok(submission) } pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } @@ -363,12 +386,19 @@ impl Outputs { impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&submission.fence()); + } submission.finish_enqueue(self.result); return; } if let Some(fallback) = self.fallback.take() { if let Err(error) = self.result { fallback.set_error(error); } - install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&Fence::from_fence(&fallback)); + } else { + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + } fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 0bfc0ba05ccd3e..f641fca3cfade5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -33,6 +33,7 @@ mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; mod sync_input; +pub(crate) mod sync_output; mod trace; mod event; mod feed; @@ -1501,7 +1502,8 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { params.features |= diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c index e6b91988c3c287..19d5b8b92f9347 100644 --- a/drivers/gpu/drm/asahi/g17/sync_input.c +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -3,31 +3,73 @@ #include #include #include "sync_input.h" +#include "sync_scope.h" + +static int collect_leaf(struct dma_fence *leaf, + struct asahi_g17_fence_leaf *out, size_t capacity, + size_t *count) +{ + if (*count == capacity) + return -ENOSPC; + out[*count].fence = dma_fence_get(leaf); + out[*count].status = dma_fence_get_status(leaf); + ++*count; + return 0; +} /* Called without the device mutex: status polling may run fence callbacks. */ int asahi_g17_collect_fences(struct dma_fence *head, struct asahi_g17_fence_leaf *out, size_t capacity) { - struct dma_fence_unwrap cursor; - struct dma_fence *leaf; - size_t count = 0; + struct dma_fence *pending[128]; + size_t count = 0, depth = 1; int error = 0; - dma_fence_unwrap_for_each(leaf, &cursor, head) { - if (dma_fence_is_container(leaf)) { - error = -EOPNOTSUPP; - break; + pending[0] = dma_fence_get(head); + while (depth && !error) { + struct dma_fence *node = pending[--depth]; + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + + if (asahi_g17_is_scope(node)) { + struct dma_fence *children[2]; + unsigned int n = asahi_g17_scope_children(node, children), i; + if (!n) { + /* Preserve a failed scope as a failed leaf, never drop its error. */ + if (dma_fence_get_status(node) < 0) + error = collect_leaf(node, out, capacity, &count); + } else if (depth + n > ARRAY_SIZE(pending)) { + for (i = 0; i < n; ++i) + dma_fence_put(children[i]); + error = -ENOSPC; + } else { + for (i = 0; i < n; ++i) + pending[depth++] = children[i]; + } + dma_fence_put(node); + continue; } - if (count == capacity) { - error = -ENOSPC; - break; + + dma_fence_unwrap_for_each(leaf, &cursor, node) { + if (asahi_g17_is_scope(leaf)) { + if (depth == ARRAY_SIZE(pending)) + error = -ENOSPC; + else + pending[depth++] = dma_fence_get(leaf); + } else if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + } else { + error = collect_leaf(leaf, out, capacity, &count); + } + if (error) + break; } - out[count].fence = dma_fence_get(leaf); - out[count].status = dma_fence_get_status(leaf); - ++count; + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + dma_fence_put(node); } - /* A bounded early exit still owns the iterator's current chain. */ - dma_fence_put(cursor.chain); + while (depth) + dma_fence_put(pending[--depth]); if (error) { while (count) dma_fence_put(out[--count].fence); diff --git a/drivers/gpu/drm/asahi/g17/sync_output.c b/drivers/gpu/drm/asahi/g17/sync_output.c new file mode 100644 index 00000000000000..08b4a95f26c9b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include "sync_output.h" +#include "sync_scope.h" + +struct asahi_g17_signal_target { + struct drm_syncobj *syncobj; + struct dma_fence_chain *chain; + u64 point; +}; + +struct asahi_g17_signal_bundle { + size_t count; + size_t anchor; + bool published; + struct asahi_g17_signal_target targets[]; +}; + +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle) +{ + size_t i; + + if (!bundle) + return; + for (i = 0; i < bundle->count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + if (target->chain) + dma_fence_chain_free(target->chain); + if (target->syncobj) + drm_syncobj_put(target->syncobj); + } + kfree(bundle); +} + +/* The Rust caller supplies its validated, immutable snapshot, never a user pointer. */ +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *syncs, + size_t count, size_t anchor, int *error) +{ + struct asahi_g17_signal_bundle *bundle; + size_t i; + + *error = -EINVAL; + if (!count || anchor >= count || !syncs[anchor].point) + return NULL; + *error = -ENOMEM; + bundle = kzalloc(struct_size(bundle, targets, count), GFP_KERNEL); + if (!bundle) + return NULL; + bundle->count = count; + bundle->anchor = anchor; + for (i = 0; i < count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + target->syncobj = syncs[i].syncobj; + if (target->syncobj) + drm_syncobj_get(target->syncobj); + if (!target->syncobj) { + *error = -ENOENT; + goto fail; + } + target->point = syncs[i].point; + if (syncs[i].timeline) { + target->chain = dma_fence_chain_alloc(); + if (!target->chain) + goto fail; + } + if (i != anchor && target->syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + /* The anchor may follow the other outputs. Reject aliases in either order. */ + for (i = 0; i < count; ++i) { + if (i != anchor && bundle->targets[i].syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + *error = 0; + return bundle; +fail: + asahi_g17_signals_free(bundle); + return NULL; +} + +static void asahi_g17_signal_install(struct asahi_g17_signal_target *target, + struct dma_fence *fence) +{ + if (target->chain) { + drm_syncobj_add_point(target->syncobj, target->chain, fence, target->point); + target->chain = NULL; + } else { + drm_syncobj_replace_fence(target->syncobj, fence); + } +} + +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted) +{ + struct dma_fence *previous, *scope; + size_t i; + + if (bundle->published) + return -EINVAL; + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + scope = asahi_g17_scope_create(previous, submitted, + bundle->targets[bundle->anchor].point); + dma_fence_put(previous); + if (!scope) + return -ENOMEM; + /* Only exported completion includes earlier work. GPU execution gets no new wait. */ + asahi_g17_signal_install(&bundle->targets[bundle->anchor], scope); + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + asahi_g17_scope_bind(scope, previous); + dma_fence_put(previous); + for (i = 0; i < bundle->count; ++i) + if (i != bundle->anchor) + asahi_g17_signal_install(&bundle->targets[i], scope); + bundle->published = true; + dma_fence_put(scope); + return 0; +} + +/* Allocation or validation failed before enqueue: install an error promise without allocating. */ +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure) +{ + size_t i; + + if (bundle->published) + return; + for (i = 0; i < bundle->count; ++i) + asahi_g17_signal_install(&bundle->targets[i], failure); + bundle->published = true; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.h b/drivers/gpu/drm/asahi/g17/sync_output.h new file mode 100644 index 00000000000000..a752d9ceaf4b0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_OUTPUT_H +#define ASAHI_G17_SYNC_OUTPUT_H +#include + +struct dma_fence; +struct drm_syncobj; +struct asahi_g17_signal_bundle; +struct asahi_g17_signal_spec { + struct drm_syncobj *syncobj; + u64 point; + u32 timeline; +}; + +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *specs, + size_t count, size_t anchor, int *error); +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle); +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted); +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs new file mode 100644 index 00000000000000..50585befb3b71f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Atomic preparation of a queue completion scope and its exported signals. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +// Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these +// local permits building this module against an older kernel's Rust bindings. +pub(crate) const SYNC_CUMULATIVE: u32 = 2; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 17; + +unsafe extern "C" { + fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, + error: *mut i32) -> *mut core::ffi::c_void; + fn asahi_g17_signals_free(bundle: *mut core::ffi::c_void); + fn asahi_g17_signals_publish(bundle: *mut core::ffi::c_void, + current: *mut bindings::dma_fence) -> i32; + fn asahi_g17_signals_fail(bundle: *mut core::ffi::c_void, + failure: *mut bindings::dma_fence); +} + +#[repr(C)] +struct SignalSpec { + syncobj: *mut core::ffi::c_void, + point: u64, + timeline: u32, +} + +/// Owns syncobj references and preallocated chain nodes; never owns user pointers. +pub(crate) struct SignalBundle(*mut core::ffi::c_void); + +impl SignalBundle { + /// C retains references before the borrowed sync items can be dropped. + pub(crate) fn new(syncs: &[crate::file::SyncItem], anchor: usize) -> Result + { + let mut specs = KVec::with_capacity(syncs.len(), GFP_KERNEL)?; + for sync in syncs { + specs.push(SignalSpec { + syncobj: sync.syncobj.as_raw().cast(), point: sync.timeline_value, + timeline: u32::from(sync.timeline), + }, GFP_KERNEL)?; + } + let mut error = 0; + // SAFETY: All entries borrow live sync objects; C copies the entries and + // acquires its own references before returning. + let raw = unsafe { asahi_g17_signals_new( + specs.as_ptr(), specs.len(), anchor, &mut error) }; + if raw.is_null() { return Err(Error::from_errno(error)); } + Ok(Self(raw)) + } + + pub(crate) fn publish(&mut self, current: &Fence) -> Result { + // SAFETY: The bundle and current fence are owned for this call; C takes + // its own references on publication and reports failure before any store. + let error = unsafe { asahi_g17_signals_publish(self.0, current.raw()) }; + if error == 0 { Ok(()) } else { Err(Error::from_errno(error)) } + } + + pub(crate) fn fail(&mut self, failure: &Fence) { + // SAFETY: Live owned arguments; this only consumes uninstalled chain nodes. + unsafe { asahi_g17_signals_fail(self.0, failure.raw()) }; + } +} + +impl Drop for SignalBundle { + fn drop(&mut self) { + // SAFETY: Exactly one owner; published syncobjs retain their own fences. + unsafe { asahi_g17_signals_free(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c new file mode 100644 index 00000000000000..55feb19a7f83d4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sync_scope.h" +#include "sync_input.h" + +struct asahi_g17_scope; +struct asahi_g17_scope_callback { + struct dma_fence_cb callback; + struct asahi_g17_scope *scope; +}; + +struct asahi_g17_scope { + struct dma_fence base; + atomic_t pending; + struct dma_fence *children[2]; + struct asahi_g17_scope_callback callbacks[2]; + struct irq_work work; + struct llist_node ready; + /* Identifies the exact anchor point this scope replaced. */ + u64 anchor_context; + u64 anchor_point; +}; + +static const struct dma_fence_ops scope_ops; +static DEFINE_PER_CPU(struct llist_head *, active_scopes); +#define SCOPE_ATOMIC_BATCH 64 + +bool asahi_g17_is_scope(struct dma_fence *fence) +{ + return fence && fence->ops == &scope_ops; +} + +static struct asahi_g17_scope *to_scope(struct dma_fence *fence) +{ + return container_of(fence, struct asahi_g17_scope, base); +} + +static const char *scope_driver_name(struct dma_fence *fence) +{ + return "asahi"; +} + +static const char *scope_timeline_name(struct dma_fence *fence) +{ + return "g17-queue-scope"; +} + +static bool scope_signaled(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + if (atomic_read_acquire(&scope->pending)) + return false; + cmpxchg(&fence->error, 1, 0); + return true; +} + +static bool scope_enable_signaling(struct dma_fence *fence) +{ + /* Child callbacks are already armed; no recursive callback installation. */ + return !scope_signaled(fence); +} + +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned long flags; + unsigned int i, count = 0; + + spin_lock_irqsave(&fence->inline_lock, flags); + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + if (scope->children[i]) + children[count++] = dma_fence_get(scope->children[i]); + spin_unlock_irqrestore(&fence->inline_lock, flags); + return count; +} + +static void scope_release_completed(struct asahi_g17_scope *scope) +{ + struct dma_fence *children[2]; + unsigned long flags; + unsigned int i; + + /* Readers take references under this lock. Signalled scopes need no history. */ + spin_lock_irqsave(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) { + children[i] = scope->children[i]; + scope->children[i] = NULL; + } + spin_unlock_irqrestore(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) + dma_fence_put(children[i]); + dma_fence_put(&scope->base); +} + +static void scope_complete(struct asahi_g17_scope *scope) +{ + cmpxchg(&scope->base.error, 1, 0); + dma_fence_signal(&scope->base); + scope_release_completed(scope); +} + +static void scope_work(struct irq_work *work) +{ + struct asahi_g17_scope *scope = container_of(work, struct asahi_g17_scope, work); + cmpxchg(&scope->base.error, 1, 0); + asahi_g17_signal_fence(&scope->base); + scope_release_completed(scope); +} + +static void scope_defer_list(struct llist_node *nodes) +{ + struct llist_node *node, *next; + llist_for_each_safe(node, next, nodes) { + struct asahi_g17_scope *scope = + container_of(node, struct asahi_g17_scope, ready); + /* Transfer the ready-list reference to the irq_work callback. */ + irq_work_queue(&scope->work); + } +} + +void asahi_g17_signal_fence(struct dma_fence *fence) +{ + struct llist_head ready = LLIST_HEAD_INIT(ready); + struct llist_node *nodes, *node, *next; + unsigned long flags; + unsigned int completed = 0; + + /* Fence callbacks are atomic. Keep the local ready list pinned while + * signaling, then finish ready scopes after the source fence is unlocked. + * New ready scopes append to this list instead of recursing or sending IPIs. + */ + preempt_disable(); + if (__this_cpu_read(active_scopes)) { + dma_fence_signal(fence); + preempt_enable(); + return; + } + __this_cpu_write(active_scopes, &ready); + dma_fence_signal(fence); + for (;;) { + /* An interrupt must not append after the final empty test. */ + local_irq_save(flags); + nodes = llist_del_all(&ready); + if (!nodes) + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + if (!nodes) + break; + llist_for_each_safe(node, next, nodes) { + if (completed == SCOPE_ATOMIC_BATCH) { + struct llist_node *more; + local_irq_save(flags); + more = llist_del_all(&ready); + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + preempt_enable(); + scope_defer_list(node); + scope_defer_list(more); + return; + } + scope_complete(container_of(node, struct asahi_g17_scope, ready)); + ++completed; + } + } + preempt_enable(); +} + +static void scope_arrived(struct asahi_g17_scope *scope, int error) +{ + if (error < 0) + cmpxchg(&scope->base.error, 1, error); + if (atomic_dec_and_test(&scope->pending)) { + struct llist_head *ready; + dma_fence_get(&scope->base); + preempt_disable(); + ready = __this_cpu_read(active_scopes); + if (ready) + llist_add(&scope->ready, ready); + preempt_enable(); + /* Foreign fence signalers have no driver completion pass to drain us. */ + if (!ready) + irq_work_queue(&scope->work); + } +} + +static void scope_callback(struct dma_fence *child, struct dma_fence_cb *callback) +{ + struct asahi_g17_scope_callback *cb = + container_of(callback, typeof(*cb), callback); + scope_arrived(cb->scope, child->error); + dma_fence_put(&cb->scope->base); +} + +static void scope_release(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + dma_fence_put(scope->children[i]); + dma_fence_free(fence); +} + +static void scope_set_deadline(struct dma_fence *fence, ktime_t deadline) +{ + struct asahi_g17_fence_leaf *leaves; + int count, i; + + if (dma_fence_is_signaled(fence)) + return; + /* A deadline is an advisory hint. Keep its traversal bounded and iterative, + * including when a foreign timeline boundary contains earlier scopes. + */ + leaves = kmalloc_array(128, sizeof(*leaves), GFP_ATOMIC | __GFP_NOWARN); + if (!leaves) + return; + count = asahi_g17_collect_fences(fence, leaves, 128); + for (i = 0; i < count; ++i) { + if (!asahi_g17_is_scope(leaves[i].fence)) + dma_fence_set_deadline(leaves[i].fence, deadline); + dma_fence_put(leaves[i].fence); + } + kfree(leaves); +} + +static const struct dma_fence_ops scope_ops = { + .get_driver_name = scope_driver_name, + .get_timeline_name = scope_timeline_name, + .enable_signaling = scope_enable_signaling, + .signaled = scope_signaled, + .release = scope_release, + .set_deadline = scope_set_deadline, +}; + +void asahi_g17_scope_bind(struct dma_fence *fence, struct dma_fence *root) +{ + struct asahi_g17_scope *scope = to_scope(fence); + /* A concurrent raw syncobj reset is invalid for the private queue anchor, + * but must not turn a hostile userspace race into a kernel dereference. + */ + if (!root || dma_fence_chain_contained(root) != fence) + return; + /* Written once before the submit returns; an unmatched snapshot takes fallback. */ + WRITE_ONCE(scope->anchor_point, root->seqno); + WRITE_ONCE(scope->anchor_context, root->context); +} + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point) +{ + struct dma_fence *contained = previous ? dma_fence_chain_contained(previous) : NULL; + struct dma_fence *parent = NULL; + struct asahi_g17_scope *scope, *prior = NULL; + u64 context; + unsigned int i; + + if (asahi_g17_is_scope(contained)) { + prior = to_scope(contained); + if (READ_ONCE(prior->anchor_context) != previous->context || + READ_ONCE(prior->anchor_point) != previous->seqno || + point <= contained->seqno) + prior = NULL; + } + if (prior) { + parent = dma_fence_get(contained); + context = contained->context; + } else { + /* Empty submissions or external edits may insert a foreign timeline point. + * Flatten that boundary once; the ordinary path retains just the prior scope. + */ + if (previous) { + parent = dma_fence_unwrap_merge(previous); + if (!parent) + return NULL; + } + context = dma_fence_context_alloc(1); + } + scope = kzalloc_obj(*scope); + if (!scope) { + dma_fence_put(parent); + return NULL; + } + dma_fence_init64(&scope->base, &scope_ops, NULL, context, point); + init_irq_work(&scope->work, scope_work); + scope->children[0] = parent; + scope->children[1] = dma_fence_get(submitted); + /* The constructor guard prevents completion while callbacks are being installed. */ + atomic_set(&scope->pending, 3); + scope->base.error = 1; + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) { + struct dma_fence *child = scope->children[i]; + if (!child) { + scope_arrived(scope, 0); + continue; + } + scope->callbacks[i].scope = scope; + dma_fence_get(&scope->base); + if (dma_fence_add_callback(child, &scope->callbacks[i].callback, scope_callback)) { + scope_arrived(scope, child->error); + dma_fence_put(&scope->base); + } + } + scope_arrived(scope, 0); + return &scope->base; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h new file mode 100644 index 00000000000000..11c84d65f59f27 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_SCOPE_H +#define ASAHI_G17_SYNC_SCOPE_H +#include +struct dma_fence; +void asahi_g17_signal_fence(struct dma_fence *fence); + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point); +void asahi_g17_scope_bind(struct dma_fence *scope, struct dma_fence *root); +bool asahi_g17_is_scope(struct dma_fence *fence); +/* Returns owned child references, or zero once the scope is signalled. */ +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]); +#endif diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 66396f89cc3339..836739ffbbebfb 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -368,6 +368,13 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. */ DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 17, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -901,6 +908,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs index a022e08223588b..72ba50c57286b4 100644 --- a/rust/kernel/drm/syncobj.rs +++ b/rust/kernel/drm/syncobj.rs @@ -15,6 +15,14 @@ pub struct SyncObj { } impl SyncObj { + /// Borrows the raw sync object for synchronous calls into DRM helpers. + /// The caller must not release the owned reference or retain this pointer + /// beyond the lifetime of `self` without taking another reference. + #[inline(always)] + pub fn as_raw(&self) -> *mut bindings::drm_syncobj { + self.ptr + } + /// Looks up a sync object by its handle for a given `File`. pub fn lookup_handle( file: &drm::File, From e12ec29f7549d845bb7f56cea0e4c7eb5a1726aa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 03:45:04 -0400 Subject: [PATCH 35/68] drm/asahi: test G17 queue budgets with firmware-reserved identities Exercise dynamic render headroom and the full logical-queue budget with zero and nonzero firmware QID prefixes. Document that the shared watchdog counter orders reservations rather than concurrent firmware exposure. (cherry picked from commit bcdfdafd80cf93fadc79df688dd0109b9c4ae8d2) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/kick.rs | 35 +++++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/port.rs | 3 +++ 2 files changed, 38 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f3183bb005df60..a1f8f70f597d47 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -307,6 +307,41 @@ mod accounting_tests { assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); } + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } + } + #[test] fn initial_reservations_yield_but_published_ids_do_not() { let mut ids = table(); diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index d8f7f4ddff8a12..9acd9ff6e5910d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -371,6 +371,9 @@ impl Port { self.generation.store(generation, Ordering::Release); } + /// Unique reservation order shared by both engines. Concurrent publishers may expose + /// their work in a different order; queue serialization and visibility use other locks. + /// The watchdog uses this number only to choose bounded cross-VM timeout grace. pub(super) fn next_publication(&self) -> u64 { self.publications.fetch_add(1, Ordering::Relaxed) + 1 } From dcbf012421f8f0941c90b2e277207a1acd86a91d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 03:46:24 -0400 Subject: [PATCH 36/68] drm/asahi: size cached compute records for every native wait Dynamic QID admission permits 128 native inputs. A 0x1000-byte record slot holds only 18 waits at offset 0x900 with 0x60 bytes each, so the nineteenth could overwrite the following live slot. Size cached slots for the complete bounded list and validate each write against its own slot before deriving an allocation offset. Keep announcement slots unchanged. Host execution of the production offset helper and regression test bodies covers all 128 waits in all 32 retained slots; the old-size control fails. (cherry picked from commit 971d4316d91cce97b92db72c4f29e5155b76d626) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 62 ++++++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 6ca63595154de9..02ae0bbbb9634a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -30,6 +30,12 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; @@ -193,8 +199,8 @@ impl Graph { let cached_records = objects.allocate( alloc, Family::QueueConfig, - RECORD_SLOTS * RECORD_STRIDE, - RECORD_SLOTS * RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, prot, cpu, )?; @@ -1196,6 +1202,32 @@ fn release_pending(released: bool, quarantined: bool, active: usize, retiring: b mod admission_tests { use super::*; + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1)); + } + #[test] fn retained_ring_pressure_waits_without_discarding_work() { let mut active = Fifo::new(); @@ -1760,6 +1792,15 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1769,17 +1810,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - let object: &mut KernelObject = if offset == ANNOUNCE { - &mut self.records + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) } else { - &mut self.cached_records + (&mut self.cached_records, CACHED_RECORD_STRIDE) }; - (object, slot * RECORD_STRIDE + offset) + (object, record_start(slot, offset, window, stride)?) } else { - (&mut *self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -2031,7 +2070,8 @@ impl Queue { for (index, wait) in native.iter().enumerate() { let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), packet.order.sequence); - let address = self.graph.record(record_slot, 0x900 + index * 0x60, 0x60, record)?; + let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, record)?; wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; From 26a14861a8a16b7edb3048db3a0c6ceeb64f2cd7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 04:00:14 -0400 Subject: [PATCH 37/68] drm/asahi: index pending compute completion identities before reading payloads The shared completion ring retains old records for active QIDs. Checking each one against every pending descriptor reads unrelated payload words and repeats a linear search, especially when a future pending command keeps the full scan open. Build a bounded per-QID kick index in the existing observation batch. Reject old kicks from the control word, then retain the exact descriptor, validity and timestamp checks. Keep all scan requests, full-ring fallbacks and retirement semantics unchanged. Add collision, descriptor-reuse, stale-payload, clear and partial-publication tests. (cherry picked from commit 9bf3039873a1e241991b5999aceff5610c660ade) Signed-off-by: DjDeveloperr --- .../drm/asahi/g17/queue/compute/retirement.rs | 156 ++++++++++++++++-- 1 file changed, 140 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b268c1c40de00c..46df3598362b75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -166,12 +166,22 @@ struct Request { descriptor: u64, kick: u64, observed: Option, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, /// Requests with no observation yet; the scan stops when this reaches zero. @@ -180,8 +190,13 @@ pub(crate) struct Batch { impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, unobserved: 0, @@ -209,45 +224,74 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); self.unobserved += 1; Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + self.fold_reader(|word| record.read(word)) + } + + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; if count == 0 { return Ok(()); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); - } let kick = (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); - }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; + } + let Some(index) = found else { return Ok(()); }; + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; if timestamps[0] == 0 || timestamps[1] == 0 || request @@ -415,3 +459,83 @@ mod tests { assert_eq!(result, Err(ENODATA)); } } + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000)).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first).unwrap(); batch.request(&second).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first).unwrap(); batch.request(&second).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t).unwrap(); batch.request(¤t).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000)).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } +} From 54d5081d9806c495fd49a3450863f662ab61d7fb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 04:16:00 -0400 Subject: [PATCH 38/68] drm/asahi: retain a deep working set of validated GPU page ranges Cache the full page footprint already checked by the VM, preserve permission-subset and mapping-epoch checks, and merge only compatible intervals. Keep the transient parser bound at 48 requests while retaining 128 intervals in the persistent cache so a depth-64 stream does not repeatedly flush every proof. Measured normal runs reduce compute depth-64 submit CPU from about 10.06 to 7.46 us and render depth-32 from 12.29 to 8.16 us. Wall throughput remains limited elsewhere. Range, permission, epoch, overflow and rotating-working-set regression tests pass. (cherry picked from commit 26446182da3aa8d98ce799478a025117b021eaba) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/validation.rs | 174 ++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b2a6cf6e8bfe37..b0f2861382ad74 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -144,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} From 56d85b5a810824d659456ed30c0fbcd8054d4003 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 05:14:09 -0400 Subject: [PATCH 39/68] drm/asahi: avoid locking pending private completion fences Frontier pruning examines independently signalled driver completion fences. Check their signalled flag before taking the generic status lock; retain the locked terminal path for errors and leave external input fence handling unchanged. Seven Neo depth-64 repetitions reduce median submit API elapsed time from 7.52 to 5.79 us. Frontier scan CPU samples fall from 1.83% to 0.25% of the machine; throughput remains about 19.5 us per command. The full benchmark matrix, candidate-bound short stress, software-fence/error gates and 20-source native fan-in pass. (cherry picked from commit b6248acf900c1cf30deac52791c2c50d10055d7b) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/job.rs | 12 ++++++++++++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 8 ++++---- drivers/gpu/drm/asahi/g17/sync_scope.c | 12 ++++++++++++ drivers/gpu/drm/asahi/g17/sync_scope.h | 2 ++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 10ba27ff049215..ea78b799085fb9 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -385,6 +385,18 @@ pub(crate) struct Fences { pub(crate) publication: Option>, } +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } +} + impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index c959f8bc8c4b89..0cc64b8b944602 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -155,7 +155,7 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } @@ -203,11 +203,11 @@ impl Queue { && !destinations.has_outputs() && self.frontiers[0].iter().all(|(sequence, fences)| !Order::contains(order.wait_through[0], *sequence) - || job::fence_status(&fences.completed) > 0) + || fences.status() > 0) && self.frontiers[1].iter().all(|(sequence, fences)| !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || job::fence_status(&fences.completed) > 0) + || fences.status() > 0) && self.failures.iter().enumerate().all(|(index, failure)| failure.as_ref().is_none_or(|(sequence, _)| !Order::contains(order.wait_through[index], *sequence))) @@ -293,7 +293,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c index 55feb19a7f83d4..a3153f3122b946 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.c +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -11,6 +11,18 @@ #include "sync_scope.h" #include "sync_input.h" +int asahi_g17_completion_fence_status(struct dma_fence *fence) +{ + /* These private fences have no signaled callback. A pending flag is + * sufficient to keep the frontier member; a concurrent signal can only + * defer pruning until the next submission. Read terminal errors through + * the normal locked API after observing the signalled flag. + */ + if (!dma_fence_test_signaled_flag(fence)) + return 0; + return dma_fence_get_status(fence); +} + struct asahi_g17_scope; struct asahi_g17_scope_callback { struct dma_fence_cb callback; diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h index 11c84d65f59f27..e88b78aacdd7b5 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.h +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -4,6 +4,8 @@ #include struct dma_fence; void asahi_g17_signal_fence(struct dma_fence *fence); +/* Only for explicitly signalled driver completion fences, with no poll op. */ +int asahi_g17_completion_fence_status(struct dma_fence *fence); struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, struct dma_fence *submitted, u64 point); From 009c101a26e91ff36ac938d4f939ff8821e1e45c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 23:19:10 -0400 Subject: [PATCH 40/68] drm/asahi: give G17 firmware restarts time to drain long work The restart handshake waited 500 one-millisecond ticks for each of its two phases. The firmware completes a restart only after the work it interrupted has drained, and with several clients running long ray-traversal jobs that takes seconds: six concurrent ray-query stress clients timed the second phase out in state 2, and the host then declared the whole device lost, killing every client including the desktop. Wait up to ten seconds per phase, the longest execution allowance a job gets, and log which phase timed out and the state the firmware was in, as well as a restart storm stop. With this the same load completes every restart and only the clients whose work was blamed or timed out see a lost device. (cherry picked from commit b08c81722d896eee8da9771e99e38e6986a2a806) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/recovery.rs | 10 +++++++++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 89f8d33bdb80c0..8aac25ab80d68d 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -24,7 +24,11 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; /// Per-QID classifications from the complete firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. @@ -192,6 +196,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -217,6 +223,8 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index cdd8cc6d5dea55..c1a2ebe232fcba 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -266,7 +266,10 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, From 6d1ae0d43de3c6df2db247dc02115996c1589247 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 23:23:44 -0400 Subject: [PATCH 41/68] drm/asahi: fail commands a GPU restart interrupted instead of completing them The firmware recovery table lists every command that was in flight when the firmware stopped: one record per command, with its queue and its completion stamp. The driver only folded the guilty and cancelled bits of those records into per-queue masks and ignored the rest. An innocent command that was interrupted part way could then still be retired by its stamps and completion record, and report success with partial output. Six concurrent ray query stress cases showed exactly that: after recoveries that cancelled nothing, three clients completed without error and with wrong results. Read the table as the packed list it is, up to its first invalid record, and mark every listed command interrupted before the firmware resumes. An interrupted command never reports success. Its failure always reaches the client, without poisoning the VM unless a recovery found it guilty; ECANCELED is no longer silent for it, and its output is not handed to consumers as valid. It is replayed only when it provably never started: a compute command with a zero-duration record, or a render group with neither its tiler start nor its fragment end written. A render group interrupted after its tiler started fails at once rather than being replayed, since blending or storage writes could apply twice. The recovery log line reports how many listed commands matched by stamp and how many fell back to the oldest command of their queue. (cherry picked from commit f1db16a83586e4bec596a71e12e0d8b87dd3b16c) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 56 ++++++++-- drivers/gpu/drm/asahi/g17/initdata.rs | 23 ++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 21 ++++ drivers/gpu/drm/asahi/g17/queue/render.rs | 103 +++++++++++++++++- .../drm/asahi/g17/queue/render/retirement.rs | 5 + drivers/gpu/drm/asahi/g17/recovery.rs | 71 +++++++++++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 11 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 20 +++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 16 +++ 10 files changed, 301 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index d692a98acea8c9..11b4c323d46680 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -270,6 +270,9 @@ pub(crate) struct Completion { /// A recovery blamed this command. Unlike `class`, which a later /// classification can change, this never clears. blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } @@ -327,6 +330,7 @@ impl Completion { replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -379,6 +383,11 @@ impl Completion { } /// Called under the device mutex as a successful retirement is queued. pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } match self.check_source_inputs() { Ok(()) => self.output.output().member.note_retired(), Err(error) => { self.note_input_error(error); } @@ -429,6 +438,9 @@ impl Completion { .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -517,6 +529,35 @@ impl Completion { self.blamed.load(Ordering::Acquire) } + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -589,6 +630,9 @@ impl Completion { if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. @@ -600,13 +644,7 @@ impl Completion { } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. - if self.spared() { - if error != ECANCELED { - self.status.report_failure(error); - } - } else if error != ECANCELED { - self.status.record(error); - } + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -640,8 +678,8 @@ impl Completion { } if self.spared() { self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 8f7551b1763c9e..0b92aa3cdd1d4e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1564,17 +1564,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 02ae0bbbb9634a..16d6351678bb73 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -718,6 +718,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3e65ea3b9a367f..1d562da115b60c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1088,6 +1088,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1271,6 +1293,9 @@ impl Pair { if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } use crate::g17::replay::{self, Decision, Group}; let qids = self.qids(); let memory = &self.memory; @@ -1295,10 +1320,11 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. + // Interrupted work is run again only if it never started. Ok(matches!( retirement::abandoned(&active.ticket, snapshot, stamps, state, false), Ok(true) - )) + ) && (!active.packet.completion.interrupted() || snapshot.untouched())) })() .unwrap_or_else(|error: Error| { failure = Some(error); @@ -1391,6 +1417,81 @@ impl Pair { Ok(true) } + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); + } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; + let mut observer = Observer { + memory: &self.memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } + } + if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(Some(true)) + } + /// The pending replay cannot proceed (closed owner, cancellation or a /// publication refusal). Its successors cannot run without it and fail with /// it. ECANCELED leaves the VM usable; any other error is the ordinary diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 24aeed6bcd0ec3..00e4997ef3661e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,11 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 8aac25ab80d68d..cfadf4bb31767e 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -30,18 +30,28 @@ const MAX_GUILTY_VMS: usize = 8; /// execution allowance (`timeout::POLL_MS` plus renewals). const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -51,9 +61,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -75,6 +98,10 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -89,7 +116,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -228,14 +257,26 @@ pub(super) fn service( return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { pr_warn!( "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", @@ -255,6 +296,26 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index c30e45e22579a9..47675c96383958 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -74,6 +74,17 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index b07423bd22115d..f50e8e350e9367 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1275,7 +1275,7 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index c1a2ebe232fcba..e13ec3e35e13cc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -131,8 +131,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -203,6 +203,22 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index f4f45a98005c33..4aaad40eee7657 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -830,6 +830,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, From 8f82617e715c231df123abfdca39af66b63fb702 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 00:00:33 -0400 Subject: [PATCH 42/68] drm/asahi: stop the G17 device on a watchdog storm only without progress Eight job watchdog expirations within 30 seconds stopped the device, even while other work kept retiring. Under overload (six concurrent ray-query stress clients) that happened within a minute: each timeout already failed its own client, yet the ninth took every other client and the desktop with it. Count retired commands and start a new storm window whenever work has retired since the previous expiration. A device that stops retiring anything still stops after eight; an overloaded one only loses the clients whose work timed out. (cherry picked from commit 30f85560fd28ce05151c16eab7a58433d125a81e) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 + drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/recovery.rs | 5 +++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 26 +++++++++++++++++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 16d6351678bb73..c0916b639ee97c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -968,6 +968,7 @@ impl Queue { } let front = self.active.front().ok_or(EIO)?; if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 1d562da115b60c..5c201ee6ec9655 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1260,6 +1260,7 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::recovery::note_retired(); crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, &front.packet, [timestamps[0], timestamps[3]]); defer(Deferred::Retired( diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index cfadf4bb31767e..84a01d975c36d5 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,6 +10,7 @@ mod attribution; mod slot; mod safety; +pub(crate) use safety::note_retired; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -152,7 +153,9 @@ impl State { restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index 47675c96383958..b97fc269143290 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -2,17 +2,41 @@ //! Fail closed on repeated watchdog/restart traffic, even across client loss. +use core::sync::atomic::{AtomicU64, Ordering}; + const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + pub(super) struct Storm { start: Option, count: u8, stopped: bool, + progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false } + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) } pub(super) fn note(&mut self, now: u64) -> bool { if self.stopped { return true; } From e1812c1a7000983a3ca78d1318ca4dbb5afd9940 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 03:50:55 -0400 Subject: [PATCH 43/68] drm/asahi: scan only the G17 compute completion records written since the last pass The completion scan walked the shared compute completion ring from its cursor until it had found a record for every pending command. A command still running has no record yet, so every pass with work in flight read all 2048 slots: about 3 us on the completion path before any fence was signalled, and the cursor never advanced while a queue stayed busy. Records are appended in order, so remember each slot's control word as of the last pass over it; the first slot that still holds it is where the firmware writes next, and the pass ends there. Stage the pending identities of every queue rather than only those the batch retires, so each new record is folded once and later batches find it latched. A record of a command published after staging, or one whose payload is still being written, ends the pass and is read again by the next one. A command whose completion stamp already covers it has its record in the ring; if a pass ends without it, the whole ring is read once and the cursor moves after the newest record. Same-boot A/B against the previous scan (us per submission, medians of two rounds, app pinned to cpu0 / cpu5): compute.light d2 33.7 -> 31.0 / 43.0 -> 33.8, d64 26.6 -> 20.4 / 14.2 -> 14.3, render.light d32 38.6 -> 34.8 / 30.4 -> 29.8; d1, render d1/d2, queue hops and 16-process compute unchanged. Event worker CPU time at compute d64 drops by a quarter. (cherry picked from commit 2c0abe2d090a9701b13b22d5ac56fb24d87cfe12) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/initdata.rs | 84 ++++++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 27 ++- .../drm/asahi/g17/queue/compute/retirement.rs | 203 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 36 +++- 4 files changed, 291 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 0b92aa3cdd1d4e..7e0c36ece43ad5 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -38,6 +38,26 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { slots } else { 0x800 } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -711,28 +731,66 @@ impl InitData { /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - /// Visits ring slots from `start`, wrapping once, until `record` reports - /// that nothing more is wanted. Returns the slot after the last one visited - /// when the scan stopped early, or `start` after a complete wrap. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( &self, start: usize, - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( + &self, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { let object = self.object(self.completion_rings[0])?; - let slots = (abi::COMPLETION_RING_SIZE / 0x40).min(0x800); - let start = start % slots; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for step in 0..slots { - let slot = (start + step) % slots; - if record(ComputeCompletion { + let mut newest = None; + for slot in 0..COMPUTE_COMPLETION_SLOTS { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })? { - return Ok((slot + 1) % slots); + }; + let control = completion.read(0)?; + seen[slot] = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } } + record(completion)?; } - Ok(start) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c0916b639ee97c..c8e00e2b563c22 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -786,33 +786,32 @@ impl Queue { || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 46df3598362b75..c2b49e7b436d77 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -161,11 +161,46 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, next: u16, } @@ -186,8 +221,18 @@ pub(crate) struct Batch { open: Option, /// Requests with no observation yet; the scan stops when this reaches zero. unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; @@ -200,6 +245,8 @@ impl Batch { slots: [(0, 0); QIDS], open: None, unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { @@ -207,12 +254,29 @@ impl Batch { self.slots.fill((0, 0)); self.open = None; self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 } /// True once every requested identity has an observation. pub(crate) fn complete(&self) -> bool { self.unobserved == 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 + } + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -238,6 +302,7 @@ impl Batch { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, next, }) .map_err(|_| ENOSPC)?; @@ -245,23 +310,37 @@ impl Batch { *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } /// Capture the control word once. Most slots belong to an older kick of a /// selected QID; reject those before reading any additional shared words. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - self.fold_reader(|word| record.read(word)) + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) } - fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; let (_, count) = self.slots[qid]; - if count == 0 { - return Ok(()); - } let kick = (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + if count == 0 { + return Ok(!self.published_after_staging(qid, kick)); + } let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; let mut descriptor = None; let mut found = None; @@ -276,7 +355,7 @@ impl Batch { } else { let value = read(6)? & !1; if value == 0 || read(7)? == 0 { - return Ok(()); + return Ok(request.observed.is_some()); } descriptor = Some(value); value @@ -289,23 +368,28 @@ impl Batch { } index = request.next; } - let Some(index) = found else { return Ok(()); }; + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); + }; let request = self.requests.get_mut(index).ok_or(EIO)?; let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); } if request.observed.is_none() { self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + /// A command published after this batch staged its queue's requests was not part of the /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { @@ -480,7 +564,7 @@ mod index_tests { #[test] fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000)).unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); let words = record(&ticket(7, 17, 0x1000), [10, 20]); batch.fold_reader(|word| { assert_eq!(word, 0); @@ -493,7 +577,7 @@ mod index_tests { let mut batch = Batch::new().unwrap(); let first = ticket(7, 33, 0x1000); let second = ticket(7, 49, 0x1000); - batch.request(&first).unwrap(); batch.request(&second).unwrap(); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); let words = record(&first, [10, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert!(!batch.complete()); @@ -508,7 +592,7 @@ mod index_tests { let mut batch = Batch::new().unwrap(); let first = ticket(7, 33, 0x1000); let second = ticket(7, 33, 0x2000); - batch.request(&first).unwrap(); batch.request(&second).unwrap(); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); let words = record(&first, [20, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); @@ -518,7 +602,7 @@ mod index_tests { fn duplicate_identity_keeps_the_first_staged_request() { let mut batch = Batch::new().unwrap(); let current = ticket(7, 33, 0x1000); - batch.request(¤t).unwrap(); batch.request(¤t).unwrap(); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); let words = record(¤t, [10, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); @@ -527,9 +611,9 @@ mod index_tests { #[test] fn clear_restarts_buckets_and_later_publication_stays_unobserved() { let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000)).unwrap(); batch.clear(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); let current = ticket(7, 49, 0x1000); - batch.request(¤t).unwrap(); + batch.request(¤t, false).unwrap(); let stale = record(&ticket(7, 33, 0x1000), [10, 20]); batch.fold_reader(|word| Ok(stale[word])).unwrap(); assert!(!batch.complete()); @@ -538,4 +622,77 @@ mod index_tests { batch.fold_reader(|word| Ok(words[word])).unwrap(); assert!(batch.complete()); } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index f50e8e350e9367..7f89a25ae6d353 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -178,6 +178,8 @@ pub(super) struct Registry { pub(super) observations: compute::retirement::Batch, /// Completion-ring slot where the next compute scan starts. completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -227,6 +229,14 @@ impl Registry { replay_outer_wait: None, observations: compute::retirement::Batch::new()?, completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -1620,25 +1630,33 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } - // Completion records are appended in order: resume after the last - // record a previous scan needed and stop once every request is found. + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { let observations = &mut registry.observations; - registry.completion_cursor = self.init.scan_compute_completions( + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( registry.completion_cursor, - |record| { - observations.fold(record)?; - Ok(observations.complete()) - }, + seen, + |record| observations.fold(record), )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } let port = &self.port; let retired = deferred.pending(); @@ -1650,7 +1668,7 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) { let busy = !queue.channel_free(); - queue.latch_observations(&mut registry.observations, masks)?; + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, From 38935eebffb029c79b671fb7452f24a45e5e4ec9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:58 -0400 Subject: [PATCH 44/68] drm/asahi: release pruned G17 frontier entries after the ioctl publishes Pruning a logical queue's frontier at the start of every enqueue dropped the last references of the commands it found finished. That tears their completions down, including the deferred timestamp cleanup, before the new command is prepared, so its doorbell waited for the teardown. Keep the finished entries until the ioctl has enqueued everything and release them then. Without room for them they are released at once, as before. Prune results, failure tracking and dependencies are unchanged. Same-boot phase timers, render.light d1 pinned to cpu5: the prune step before publication 857 -> 269 ns per submit; the teardown now follows the doorbell. (cherry picked from commit 39bdd5b8e2d584fff0d342a71a8945e64c8a9673) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 0cc64b8b944602..16891524e25a06 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -95,6 +95,9 @@ pub(crate) struct Queue { proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. @@ -133,6 +136,7 @@ impl Queue { proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + finished: KVec::new(), failures: [None, None], _slot: slot, }) @@ -154,11 +158,20 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { let status = fences.status(); if status == 0 { return true; } + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); + } // One earliest failed ordinal covers every later prefix without // retaining its packet, mappings, or every failed fence. if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { @@ -461,6 +474,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { From 06c3e949f488facfefcb4636d7a04d67e84f75ac Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 45/68] drm/asahi: publish both G17 render QoS owners in one pass Each render publication published its fragment and tiling QoS owners one after the other. Every publish read back the row it had just stored byte by byte and read-modify-wrote the submission counters, all in the uncached QoS table, and the second publish read words in the cache lines the first had just written, so its reads waited for those stores to drain. Publish a command's owners together: check every owner first, read each table word once before the first store, then store the identities, one fence, and the counters. The firmware-visible values and their order per owner are those of one publish per owner; a refused publication now changes nothing instead of being cancelled. Compute keeps its single owner and gets the same read-before-store pass. Same-boot phase timers, pinned to cpu5: render QoS publication 1041 + 265 -> 262 ns per submit; compute ioctl entry to doorbell 9.05 -> 8.44 us. (cherry picked from commit b41ecb56c0775a43b747adaa37661b353651dbff) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/port.rs | 11 ++ drivers/gpu/drm/asahi/g17/qos.rs | 143 ++++++++++++------ drivers/gpu/drm/asahi/g17/queue/render.rs | 17 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 8 +- .../drm/asahi/g17/runtime/render/direct.rs | 8 +- 5 files changed, 127 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 9acd9ff6e5910d..45ddae97193632 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -333,6 +333,17 @@ impl Port { self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) } + /// Publishes the two owners of a render command, fragment then tiling. + pub(super) fn qos_publish_pair( + &self, + owners: [qos::Owner; 2], + scheduler: u64, + policy: Policy, + ) -> Result<[qos::Publication; 2]> { + let view = self.qos_view()?; + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + } + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { let view = self.qos_view()?; self.qos.lock().cancel(&view, publication) diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 4f6913ad5ee95c..b36656b0174a7b 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -53,6 +53,14 @@ impl<'a> View<'a> { fn row(&self, qid: usize) -> [u8; 8] { core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) } + /// The complete row in one access. The table is uncached: a publication reads + /// each row once, before it stores anything. + fn row_word(&self, qid: usize) -> [u8; 8] { + let row = core::ptr::from_ref(&self.rows[qid]).cast::(); + // SAFETY: Rows are eight bytes at eight-byte offsets of the eight-byte + // aligned table (checked in from_raw), so the row is one aligned word. + unsafe { (*row).load(Ordering::Relaxed) }.to_le_bytes() + } } #[derive(Clone, Copy, Debug)] @@ -108,66 +116,111 @@ impl Accounting { policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { - let (qid, qos, dm) = owner.indices()?; + let [publication] = self.publish_owners(view, [owner], scheduler, policy, clock)?; + Ok(publication) + } + + /// Publishes the owners of one command's queues, in order, with the same + /// firmware-visible values as one `publish` per owner. The owners use + /// distinct queues and data masters. Every check precedes the first store, + /// so a refusal changes nothing; every table word is read before the first + /// store, so no uncached read waits for an earlier store to drain. + pub(crate) fn publish_owners( + &mut self, + view: &View<'_>, + owners: [Owner; N], + scheduler: u64, + policy: Policy, + clock: impl FnOnce() -> (u64, u64), + ) -> Result<[Publication; N]> { if scheduler == 0 { return Err(EINVAL); } - let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; - let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; - let starts_busy = self.dm_in_flight[dm] == 0; - let time = if starts_busy { + let mut indices = [(0, 0, 0); N]; + for (index, owner) in owners.iter().enumerate() { + indices[index] = owner.indices()?; + let (qid, _, dm) = indices[index]; + if indices[..index] + .iter() + .any(|&(other_qid, _, other_dm)| other_qid == qid || other_dm == dm) + { + return Err(EINVAL); + } + self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + } + let rows = indices.map(|(qid, _, _)| view.row_word(qid)); + let queue_submitted = indices.map(|(qid, _, _)| view.queue_submitted[qid].load(Ordering::Relaxed)); + let owner_submitted = indices.map(|(_, qos, _)| view.owner_submitted[qos].load(Ordering::Relaxed)); + let dm_submitted = indices.map(|(_, _, dm)| view.times_and_counts[3 + dm].load(Ordering::Relaxed)); + let starts_busy = indices.map(|(_, _, dm)| self.dm_in_flight[dm] == 0); + let busy = starts_busy.iter().any(|&starts| starts); + let intervals = if busy { view.intervals.load(Ordering::Relaxed) } else { 0 }; + for (index, owner) in owners.iter().enumerate() { + let (qid, _, _) = indices[index]; + if self.in_flight[qid] != 0 + && (rows[index][0] != owner.qos || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + } + let time = if busy { let (counter, frequency) = clock(); time_from_host(counter, frequency)? } else { 0 }; - if self.in_flight[qid] != 0 - && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master) - { - return Err(EBUSY); + let class = policy.qos_class(); + let mut first = false; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, _) = indices[index]; + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(class, Ordering::Relaxed); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); + first |= self.in_flight[qid] == 0; } - view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); - let record = view.row(qid); - view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(policy.qos_share(), Ordering::Relaxed); - if self.in_flight[qid] == 0 { + if first { fence(Ordering::SeqCst); } - view.queue_submitted[qid].store( - view.queue_submitted[qid] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - view.owner_submitted[qos].store( - view.owner_submitted[qos] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - if starts_busy { - view.times_and_counts[dm].store(time, Ordering::Relaxed); - } - let submitted = &view.times_and_counts[3 + dm]; - let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); - submitted.store(dm_submitted, Ordering::Relaxed); - if starts_busy { - view.intervals.store( - view.intervals.load(Ordering::Relaxed).wrapping_add(1), + let mut publications = [Publication { + record: [0; 8], + owner: owners[0], + dm_submitted: 0, + }; N]; + let mut intervals = intervals; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, dm) = indices[index]; + view.queue_submitted[qid].store(queue_submitted[index].wrapping_add(1), Ordering::Relaxed); + // Owners of one logical queue share their submission count. + let earlier = indices[..index].iter().filter(|&&(_, other, _)| other == qos).count(); + view.owner_submitted[qos].store( + owner_submitted[index].wrapping_add(earlier as u32 + 1), Ordering::Relaxed, ); + if starts_busy[index] { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = dm_submitted[index].wrapping_add(1); + view.times_and_counts[3 + dm].store(submitted, Ordering::Relaxed); + if starts_busy[index] { + intervals = intervals.wrapping_add(1); + view.intervals.store(intervals, Ordering::Relaxed); + } + let mut record = rows[index]; + record[0] = owner.qos; + record[1] = class; + self.in_flight[qid] += 1; + self.dm_in_flight[dm] += 1; + self.dm_by_qid[qid] = owner.data_master; + publications[index] = Publication { + record, + owner: *owner, + dm_submitted: submitted, + }; } - self.in_flight[qid] = next_qid; - self.dm_in_flight[dm] = next_dm; - self.dm_by_qid[qid] = owner.data_master; fence(Ordering::SeqCst); - Ok(Publication { - record, - owner, - dm_submitted, - }) + Ok(publications) } /// A possibly exposed producer must retain its reference until completion. diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 5c201ee6ec9655..e8540faa620a19 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -83,12 +83,13 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; @@ -800,10 +801,12 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4aaad40eee7657..b5649f059f5d2c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -531,13 +531,13 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - self.firmware.port.qos_publish(owner, scheduler, policy) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.firmware.port.qos_cancel(publication) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index cd7bad6ecce5d6..dd3a9c881a142a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -53,13 +53,13 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - self.port.qos_publish(owner, scheduler, policy) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.port.qos_cancel(publication) From 46b125c151a36027e8dad1006846cd414c0f2210 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 46/68] drm/asahi: stop copying G17 fragment registers to the unused client page Every render preparation zeroed the pair's client register page (one uncached 16 KiB page) and copied the fragment descriptor's register arrays into it. Nothing reads that page: kick entries bind the register arrays embedded in the descriptor (register_bindings), and the page is kept only for its allocation position. Keep the array bounds check and drop the copy. Same-boot phase timers, render.light d1 pinned to cpu5: 426 -> 18 ns per submit. (cherry picked from commit b86c445ba2829f644075ca663217abe65844e8c0) Signed-off-by: DjDeveloperr --- .../gpu/drm/asahi/g17/queue/render/memory.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 80ee0b871cc829..b14ccbf879b5f8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -571,21 +571,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, From 6b652b2dde0011a7b321ec757452b0346ece684a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 47/68] drm/asahi: read the G17 render work rings once per direct publication The direct render path read both render work rings' cursors (uncached) for the room check, again to write the pair's records and again to expose them, all under the render publication lock. No other host producer moves while that lock is held and the firmware consumers only advance, so the first reading stays valid. Return it from the room check and use its producers for the write and the exposure. Same-boot phase timers, render.light d1 pinned to cpu5: record write and exposure 435 + 161 -> 410 + 113 ns per submit. (cherry picked from commit 5a55068193e213d7115a63f2222ffc304a4ced98) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/port.rs | 75 +++++++++++++------ .../drm/asahi/g17/runtime/render/direct.rs | 17 ++++- 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 45ddae97193632..8d129c49c640cd 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -103,6 +103,22 @@ pub(super) struct Port { submissions: AtomicU64, } +/// Room in both render work rings of one priority, observed under the render publication +/// lock. While that lock is held no other host producer moves, and the firmware consumers only +/// advance, so the observed producers stay current and the room stays available. +#[derive(Clone, Copy)] +pub(super) struct RenderRoom { + priority: u8, + /// Fragment, then tiling producer. + producers: [u32; 2], +} + +impl RenderRoom { + pub(super) fn priority(&self) -> u8 { + self.priority + } +} + /// A compute work-ring slot reserved before any command becomes visible. Dropping an unused /// claim returns the slot. pub(super) struct OuterClaim<'a> { @@ -225,20 +241,33 @@ impl Port { )) } - /// Both render work rings of `priority` have a free slot. Only firmware consumption, - /// never a publisher without the render lock, changes this. - pub(super) fn render_room(&self, _guard: &MutexGuard<'_, ()>, priority: u8) -> Result { + /// Reads both render work rings of `priority` once. Returns their producers if both have a + /// free slot; the room holds for as long as `guard`. Only firmware consumption, never a + /// publisher without the render lock, changes it. + pub(super) fn observe_render_room( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + ) -> Result> { let (fragment, tiling) = self.render_rings(priority)?; - Ok(fragment.free()? > 0 && tiling.free()? > 0) + let cursors = [fragment.cursors()?, tiling.cursors()?]; + let room = |[first, second, producer]: [u32; 3]| { + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + room(first).min(room(second)) + }; + Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + })) } - /// Copies a render pair's fragment, then tiling record into the slots after each - /// producer without exposing them, as `InitData::publish_work_pair` does for a deferred - /// pair. Returns the producers that expose them. + /// Copies a render pair's fragment, then tiling record into the slots after the producers + /// that `room` observed, without exposing them, as `InitData::publish_work_pair` does for a + /// deferred pair. Returns the producers that expose them. pub(super) fn write_render_pair( &self, _guard: &MutexGuard<'_, ()>, - priority: u8, + room: &RenderRoom, fragment: &WorkSlot, tiling: &WorkSlot, commit_accounting: impl FnOnce() -> Result, @@ -249,21 +278,20 @@ impl Port { { return Err(EINVAL); } - let (fragment_ring, tiling_ring) = self.render_rings(priority)?; - let slot = |ring: &RingView| -> Result<(*mut u8, u32)> { - let [first, second, producer] = ring.cursors()?; - let next = (producer + 1) % WORK_RING_SLOTS; - if next == first || next == second { + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + let slot = |ring: &RingView, producer: u32| -> Result<(*mut u8, u32)> { + if producer >= WORK_RING_SLOTS { return Err(EINVAL); } // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. - Ok((unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }, next)) + let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + Ok((slot, (producer + 1) % WORK_RING_SLOTS)) }; - let (fragment_slot, fragment_next) = slot(&fragment_ring)?; - let (tiling_slot, tiling_next) = slot(&tiling_ring)?; + let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; + let (tiling_slot, tiling_next) = slot(&tiling_ring, room.producers[1])?; let copy = |record: &WorkSlot, destination: *mut u8| { - // SAFETY: The slot after the producer is host-owned until the producer moves past - // it, and the render lock excludes every other host writer of this ring. + // SAFETY: The room proves that the slot after the producer is host-owned until the + // producer moves past it, and the render lock excludes every other host writer. unsafe { core::ptr::copy_nonoverlapping( core::ptr::from_ref(record).cast::(), @@ -281,17 +309,16 @@ impl Port { Ok([fragment_next, tiling_next]) } - /// Exposes a pair written by `write_render_pair`, fragment first, as - /// `InitData::publish_retained_work_pair` does: the producers must still precede `next`. + /// Exposes a pair written by `write_render_pair` with `room`, fragment first, as + /// `InitData::publish_retained_work_pair` does: `next` must follow the observed producers. pub(super) fn expose_render_pair( &self, _guard: &MutexGuard<'_, ()>, - priority: u8, + room: &RenderRoom, next: [u32; 2], ) -> Result { - let (fragment_ring, tiling_ring) = self.render_rings(priority)?; - for (ring, next) in [(&fragment_ring, next[0]), (&tiling_ring, next[1])] { - let [_, _, producer] = ring.cursors()?; + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + for (producer, next) in room.producers.into_iter().zip(next) { if next != (producer + 1) % WORK_RING_SLOTS { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index dd3a9c881a142a..1f42d1e6f404f9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -14,7 +14,7 @@ use super::super::{Backend, DeferredBatch}; use crate::g17::{ job::Packet, object::Allocator, - port::Port, + port::{Port, RenderRoom}, queue::render::{Pair, PublicationLease, RenderAddresses}, }; use core::sync::atomic::Ordering; @@ -39,6 +39,7 @@ pub(in crate::g17::runtime) struct Loan { struct DirectHost<'a> { port: &'a Port, render: &'a MutexGuard<'a, ()>, + room: RenderRoom, generation: u64, activation: Option<(u16, u64)>, } @@ -79,8 +80,11 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]> { + if priority != self.room.priority() { + return Err(EINVAL); + } self.port - .write_render_pair(self.render, priority, fragment, tiling, commit, late_tiling) + .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { // The lease leaves preparation when the pair is returned; nothing can @@ -95,7 +99,10 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { _tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], ) -> Result { - self.port.expose_render_pair(self.render, priority, next) + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port.expose_render_pair(self.render, &self.room, next) } /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { @@ -302,9 +309,10 @@ impl Backend { }; pair.prepare(&alloc, packet, loan.addresses)?; let render = port.render_lock(); - if !lease.is_current() || !port.render_room(&render, priority)? { + if !lease.is_current() { return Err(EBUSY); } + let room = port.observe_render_room(&render, priority)?.ok_or(EBUSY)?; let mut dependencies = pair.own_dependencies(self.owner, packet)?; if let Some(footprint) = packet.footprint.as_ref() { dependencies.hazard_count = self.admission.publication_barriers( @@ -317,6 +325,7 @@ impl Backend { let mut host = DirectHost { port, render: &render, + room, generation, activation: None, }; From f68808add2faffc12e833a1ed8a5105ad74186d8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 48/68] drm/asahi: request the G17 tiling descriptor slot before preparing a render Clearing a render's tiling descriptor slot took about 480 ns for 2.5 KiB while clearing the 8.7 KiB fragment slot took 80 ns: the slot was last used by the firmware for an earlier command, and every line had to be fetched for writing one at a time. Issue store prefetches for the slot at the start of preparation, so the transfers overlap the target, scene and generation preparation that precedes the descriptor write. Same-boot phase timers, render.light d1 pinned to cpu5: tiling descriptor clear 483 -> 52 ns, ioctl entry to doorbell 12.04 -> 11.39 us. Same-boot A/B of this series (exp_mask diagnostic restoring each old path, 3 interleaved rounds, gpubench --runs 5 --run-ms 250, us/submit): render.light d1/d2/d32 cpu5 126.5/48.5/31.7 -> 106.1/46.7/30.8, cpu0 104.7/46.6/44.7 -> 103.6/46.0/37.5 (d32 cpu0 noisy); render ioctl entry to doorbell 14.4 -> 12.1 us (phase timers). compute.light, render queues d2/4/8, render procs d2/16, copy.bw buf2img d1/4 and sync.signal hop unchanged within noise. (cherry picked from commit b67018ed5974069c13c13249834dbc3b84e806ed) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index e8540faa620a19..b9f179495ab99d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -374,6 +374,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index b14ccbf879b5f8..e7d835e15eb476 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -537,6 +537,26 @@ const _: () = { }; impl Memory { + /// Requests the tiling descriptor slot of `ordinal` for writing. The firmware accessed + /// that slot for the command that used it last, so the host no longer holds its lines; + /// requesting them before the rest of the preparation overlaps those transfers with it + /// instead of stalling the descriptor write one line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + const LINE: usize = 64; + let size = size_of::(); + let Ok(base) = self.graph.descriptors.pointer(Graph::offset(0, ordinal), size) else { + return; + }; + for offset in (0..size).step_by(LINE) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(offset), + options(nostack, preserves_flags, readonly)); + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, From 0eb9a8f6a1551e495f2f1d6abde7508e27b9c867 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 49/68] drm/asahi: store G17 register arrays without an intermediate copy Without scratch storage a render's first tiling and fragment register arrays drop their SCS-enable entry. Each was copied into a trimmed array on the stack, initialised entry by entry, and then copied again into the descriptor. Store the remaining entries straight into the descriptor's array instead; the stored entries, size and address are unchanged. Same-boot phase timers, render.light d1 pinned to cpu5: tiling register store 310 -> 45 ns, fragment array 0 store 245 -> 69 ns, ioctl entry to doorbell 11.15 -> 10.77 us. (cherry picked from commit 4d0362d0c18d288d3bd520fdfb69c2e7624cf6b0) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/fw/kick.rs | 14 ++++++++++++++ drivers/gpu/drm/asahi/g17/fw/render.rs | 9 ++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 4f3a70671eb87d..ca2b6604a302b5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e270d40392a88..aa8693bf7140b7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1110,9 +1110,7 @@ impl TaDescriptor { } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1456,10 +1454,7 @@ impl FragmentSlot { } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; From 10592352562c19315208557611a238b9a6fa5b1d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 50/68] drm/asahi: also request the G17 queue configuration records early A render's two queue configuration records are rewritten for every command that carries one, in records the firmware accessed for an earlier command, the same pattern as the tiling descriptor slot. Request them for writing together with that slot at the start of preparation. Same-boot phase timers, render.light d1 pinned to cpu5: fragment prefix and kick 219 -> 176 ns, record write 328 -> 271 ns, ioctl entry to doorbell 10.77 -> 10.55 us. (cherry picked from commit 861b39d948f6c0588b3646e1246e660b52230e5b) Signed-off-by: DjDeveloperr --- .../gpu/drm/asahi/g17/queue/render/memory.rs | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index e7d835e15eb476..756f96e3fd048d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -537,22 +537,30 @@ const _: () = { }; impl Memory { - /// Requests the tiling descriptor slot of `ordinal` for writing. The firmware accessed - /// that slot for the command that used it last, so the host no longer holds its lines; - /// requesting them before the rest of the preparation overlaps those transfers with it - /// instead of stalling the descriptor write one line at a time. + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. pub(super) fn prefetch_descriptors(&self, ordinal: u64) { - const LINE: usize = 64; - let size = size_of::(); - let Ok(base) = self.graph.descriptors.pointer(Graph::offset(0, ordinal), size) else { - return; - }; - for offset in (0..size).step_by(LINE) { - // SAFETY: A prefetch hint inside the retained mapping; it has no - // architectural effect on memory. - unsafe { - core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(offset), - options(nostack, preserves_flags, readonly)); + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } } } } From 39ccfb2302b42c158314d07515478dda9c527fbc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 51/68] drm/asahi: scan only render pool slots for G17 free-list leases Render free-list leases live only in slots from FIRST_RENDER_POOL up; Pools::slot() refuses every lower index. The release observation each render preparation performs, the release publication and the linger expiry still walked all 256 entries. Walk the render range only. Phase timers, render.light d1 pinned to cpu5 (two boots): lease observation 346 -> 213 ns per submit. (cherry picked from commit 66ef051d8bd4e66bdc145eb7fb7a920235a8e668) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime/render/pools.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 78e5e3a219358b..1e3354d90dbfb9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -179,7 +179,7 @@ impl Pools { ) -> (bool, Option) { let mut ended = false; let mut next: Option = None; - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(since) = lease.lingering() else { continue; }; @@ -212,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -225,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; From 13ce97531c9a4cbc89385b73949911423aefb842 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 07:49:37 -0400 Subject: [PATCH 52/68] drm/asahi: omit G17 render queue configurations that only reload the held completion A render published onto a pair whose queues have drained seeds each queue's completion value, and the seed forces a queue configuration record onto both item rings. When the previous publication of the pair in the same recovery epoch was the last kick of each queue, the queue already holds exactly the value the seed would load, so the record carries nothing new. Keep the completion values of each successful publication with the configuration cache and publish the record only when its identity changed or the seed differs from the held value. Recovery, replay and graph rebuilds drop the kept values, so those publications still seed. The firmware processes each record before starting the kick. On Neo it shortened the kick of a render on a drained pair by about 1.5 us per record (start-to-start 13.5 -> 10.7 us in render.light queues d4). Same-boot A/B, alternated per row, 3 rounds, medians in us per submit: render.light queues d2 54.70 -> 51.88 (-5.2%) render.light queues d4 45.25 -> 42.15 (-6.8%) render.light queues d8 41.65 -> 38.79 (-6.9%) render.light procs d2 54.68 -> 51.84 (-5.2%) render.light procs d16 40.84 -> 37.63 (-7.9%) Deep single-queue rows publish onto busy pairs, which never seed, and are unchanged. (cherry picked from commit 1aec5be232da7bacbbe38b7f9b22c27906eac8c6) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/render.rs | 31 ++++++++++++++++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index b9f179495ab99d..9469d526f49c8e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -192,6 +192,9 @@ pub(crate) struct Pair { /// An owner change in flight: the previous scheduler page and the ordinal of /// the new owner's first command, until that command leaves the pair. switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] @@ -290,6 +293,7 @@ impl Pair { context_update: true, executed: 0, switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -649,10 +653,26 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -865,6 +885,7 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -1142,6 +1163,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1417,6 +1439,7 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); + self.published_completions = None; self.replay.push(packet).map_err(|_| EIO)?; } Ok(true) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 209ce0b8176ecb..81d4306eda28c5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -252,6 +252,7 @@ impl ParkedPair { context_update: true, executed: self.executed, switch: None, + published_completions: None, })) } } From 5d7f4a718ea991b64e4ed340206c9d4ebaf6e4fd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 08:31:47 -0400 Subject: [PATCH 53/68] drm/asahi: keep one CPU mapping per G17 timestamp buffer Every timestamped command mapped the destination buffer into the kernel for each timestamp it carried: owned_vmap() takes the object's reservation lock to count a mapping user. Because dropping that mapping takes the lock too, which fence producers must not wait for, each command also allocated a cleanup work item and queued it when its completion was released, waking a worker per command. Map a timestamp buffer once, when it is bound, and keep that mapping for the buffer's lifetime. A command's destination keeps a reference to the buffer and an offset, so releasing it takes no lock. The buffer's own last reference can still be a command's, released where fence producers run, so the buffer defers the release of its mappings to the cleanup queue; that happens once per buffer instead of once per command. The mapping is made outside the device mutex. Destination validation and the write itself are unchanged; G16's CPU-retired buffers keep their per-command mapping. Same-boot phase timers (exp_mask diagnostic restoring the old path), render.light d1 pinned to cpu5: timestamp destination resolve 410 -> 132 ns, completion teardown at the end of the ioctl 578 -> 153 ns, ioctl 16.9 -> 16.2 us with the timers; compute.light d1 resolve 367 -> 152 ns; query kind=occlusion d1 resolve 337 -> 141 ns, teardown 636 -> 162 ns. Same-boot ABBA, 4 rounds (query occlusion d1/d16, query timestamp d1/d16, render.light d1/d2/d32, compute.light d1/d2/d64, buf2img d1, sync.signal hop, idle.wake), pinned to cpu5: unchanged within noise. Regression gate passes; deqp-vk query_pool.*, synchronization2 basic and timeline_semaphore, api.command_buffers and renderpass subsets: 22847 pass, 791 not supported, 0 fail. (cherry picked from commit d0df244b8f023037bb0f9cc9793f1d0f285cfc87) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/completion.rs | 36 ++++------ drivers/gpu/drm/asahi/g17/mod.rs | 28 ++++---- drivers/gpu/drm/asahi/mmu.rs | 93 ++++++++++++++++++------- drivers/gpu/drm/asahi/timestamp.rs | 44 ++++++++++-- 4 files changed, 135 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 11b4c323d46680..02e7bbc5c33dc8 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -27,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +61,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,22 +83,13 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) } - pub(crate) fn has_outputs(&self) -> bool { self.0.is_some() } + pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f641fca3cfade5..a0d23c9d69e1e8 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1641,18 +1641,22 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - crate::timestamp::Buffer::firmware(bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )?) + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index cda3ac8d206bd7..7e8945638438ad 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1229,56 +1229,83 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, - offset: usize, -} +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); -impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: gem.owned_vmap::<0>()?, - offset, - }) + self.0.owner().size() } - pub(crate) fn write(&self, value: u64) { + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + return; + } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.mapping.owner().timestamp_lock.lock(); - // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. - let base = self.mapping.addr.cast::(); - let pointer = unsafe { base.add(self.offset) }; + let _guard = self.0.owner().timestamp_lock.lock(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let base = self.0.addr.cast::(); + let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // SAFETY: All eight bytes were checked above; the vmap pins the backing. // A naturally aligned query result becomes available in one complete store. let destination = unsafe { &*pointer.cast::() }; destination.store(value.to_le(), Ordering::Relaxed); } else { for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(self.offset + index).cast::() }; + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; destination.store(byte, Ordering::Relaxed); } } // A CPU-retired query can be consumed by a later GPU command. WRITEBACK // objects use a cacheable vmap, while GPU mappings need not snoop that cache. // Publish the bytes before signalling the completion fence or queue frontier. - if self.mapping.owner().is_writeback() { + if self.0.owner().is_writeback() { crate::pgtable::clean_poc(pointer, size_of::()); } crate::pgtable::poc_barrier(); } } +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +pub(crate) struct TimestampMapping { + mapping: TimestampVmap, + offset: usize, +} + +impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1314,6 +1341,13 @@ impl KernelMapping { /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1324,7 +1358,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - TimestampMapping::new(gem, start) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 976ec9f6bf48be..0d365a8d930bbf 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -13,10 +13,20 @@ use kernel::{prelude::*, sync::Arc}; #[cfg(CONFIG_DRM_ASAHI_G16)] use kernel::{uapi, xarray}; +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. /// CPU-retired timestamps retain their backing without consuming global firmware address space. pub(crate) enum Buffer { - Firmware(Arc), + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), #[cfg(CONFIG_DRM_ASAHI_G16)] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. @@ -27,7 +37,11 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) } /// Retain a CPU-written byte range without publishing any firmware address. @@ -46,7 +60,7 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { - Self::Firmware(mapping) => mapping.size(), + Self::Firmware(buffer) => buffer.mapping.size(), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { range, .. } => range.end - range.start, } @@ -54,7 +68,7 @@ impl Buffer { pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { - Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -65,10 +79,30 @@ impl Buffer { } } + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. pub(crate) fn firmware_mapping(&self) -> Result> { match self { - Self::Firmware(mapping) => Ok(mapping.clone()), + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { .. } => Err(EINVAL), } From 1a3fd7f8a6cc7dbb3c300ea360d72b5b0f24a660 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 12:55:25 -0400 Subject: [PATCH 54/68] drm/asahi: withhold G17 compute doorbells behind two unretired commands Each compute publication rang its pipe doorbell. The firmware handles a doorbell by reading the whole work ring of the priority class, and a doorbell that arrives while it finishes a command delays that command's completion: writing its end timestamp and stamp, and releasing the commands that wait for it. In the cross-queue hop the stamp followed the GPU end by 11.5 us without a doorbell in that window and by about 24 us with one. Three quarters of the completions had one, because each hop publishes two commands. A command with two or more unretired commands of its queue ahead cannot be the next one to start. Withhold its doorbell, and ring the withheld doorbell from the compute scan once at most one unretired command precedes the first publication it covers. A doorbell rung meanwhile on the same ring announces the work slot earlier. Shallow queues ring as before. Same-boot A/B on Neo, alternated per row, 3 rounds, unpinned, medians: sync.signal queue timeline 30.13 -> 23.60 us per hop (-21.7%) sync.signal queue binary 29.14 -> 23.31 us per hop (-20.0%) compute.light d64 14.55 -> 13.91 us (-4.4%) compute.light d1/d2 unchanged (103.0 / 34.0 us) render.light d1/d32 unchanged Firmware side, from the completion words: GPU end to stamp went from a median of 24.3 us to 12.0 us, and doorbells per command from 1.0 to 0.2. (cherry picked from commit 3ff8c49c5f12b908a72e6c076ff0d7336ffddcb8) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 41 ++++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 10 ++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c8e00e2b563c22..5359a732562c4d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -369,6 +369,10 @@ use crate::g17::{ }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; +/// Position in its queue's active ring from which a publication withholds its doorbell: +/// with two unretired commands ahead, it cannot be the next to start. +const WITHHOLD_DEPTH: usize = 3; + struct Fifo { entries: [Option; IN_FLIGHT], head: usize, @@ -589,9 +593,22 @@ pub(crate) struct Queue { retirement_proved: bool, source_readers: Arc, source_hint: crate::g17::sync_input::ReaderHint, + /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more + /// unretired commands of this queue ahead of them, and the publication of the first. + withheld_doorbell: Option<(u64, u64)>, } impl Queue { + /// Returns the withheld doorbell once at most one unretired command of this queue + /// precedes the first publication it covers. + pub(crate) fn due_doorbell(&mut self) -> Option { + let (doorbell, first) = self.withheld_doorbell?; + if self.active.iter().filter(|active| active.published < first).count() > 1 { + return None; + } + self.withheld_doorbell = None; + Some(doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( @@ -656,6 +673,7 @@ impl Queue { retirement_proved: false, source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, source_hint: crate::g17::sync_input::ReaderHint::new(), + withheld_doorbell: None, }) } pub(crate) fn qid(&self) -> u8 { @@ -1599,6 +1617,7 @@ impl Queue { ring: DescriptorRing, ) -> Result<(u16, u64)> { if !self.can_rearm() { return Err(EAGAIN); } + self.withheld_doorbell = None; self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -2226,12 +2245,30 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - // Every publication rings its own pipe and priority class, as render + // A publication rings its own pipe and priority class, as render // publication does: pipe 2 (compute) in bits 1:0, priority class in // bits 3:2. The generic work doorbell left the firmware to find the // ring itself and delayed dispatch. + // + // A doorbell makes the firmware read the whole work ring of its class, + // and each one it handles while finishing a command delays that + // command's completion and the release of its waiters. A command with + // two or more unretired commands of its queue ahead cannot start soon: + // its doorbell is withheld until at most one precedes it (see + // `due_doorbell`), and any doorbell rung meanwhile on the same ring + // announces its work slot earlier. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; + if self.active.len >= WITHHOLD_DEPTH + && self.withheld_doorbell.is_none_or(|(withheld, _)| withheld == doorbell) + { + self.withheld_doorbell.get_or_insert((doorbell, published)); + } else { + host.notify(doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _)| withheld == doorbell) { + self.withheld_doorbell = None; + } + } crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 7f89a25ae6d353..625339afafeaaf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1661,6 +1661,7 @@ impl super::Firmware { let port = &self.port; let retired = deferred.pending(); let mut freed = false; + let mut doorbells = [0u64; 4]; for mut queue in registry .compute .iter_mut() @@ -1677,6 +1678,15 @@ impl super::Firmware { &mut |event| deferred.push(event), )?; freed |= busy && queue.channel_free(); + // Withheld doorbells are rung once their command is next but one; one per + // priority class announces every queue's work slots on that ring. + if let Some(doorbell) = queue.due_doorbell() { + let class = ((doorbell >> 2) & 3) as usize; + doorbells[class] = doorbell; + } + } + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + self.primary.notify(doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. From 26ee6b90624cb65282cf025ee572c6eb6e4a6918 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 13:07:37 -0400 Subject: [PATCH 55/68] drm/asahi: drop G17 compute doorbells that another doorbell announced A withheld compute doorbell is rung once its command is next but one. When a doorbell of the same priority class was rung after the last withheld work slot was written, the firmware has already read that slot, so the withheld doorbell repeats an announcement it has handled. Count the compute work slots written per class, and note the count read before each compute doorbell is rung. A due withheld doorbell whose slots are covered by that count is dropped instead of rung. Same-boot A/B on Neo against the withholding alone, alternated per row, 3 rounds, unpinned, medians: sync.signal queue timeline 23.47 -> 22.70 us per hop (-3.3%) sync.signal queue binary 23.41 -> 23.02 us per hop (-1.7%) compute.light d1/d2/d64, render.light d1/d32: unchanged Doorbells per command in the hop went from 0.22 to 0.16. (cherry picked from commit b1f5dc6df0ba5650c8424027a025db233c6544c4) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/port.rs | 32 ++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime.rs | 16 +++++++++-- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 8d129c49c640cd..db043ef9d6f91e 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -98,6 +98,11 @@ pub(super) struct Port { generation: AtomicU64, /// Device publication order shared by compute and render commands. publications: AtomicU64, + /// Compute work slots written per priority class. + compute_written: [AtomicU64; PRIORITIES], + /// `compute_written` as read before the latest compute doorbell of each class: every slot + /// counted there was in its ring when the firmware handled that doorbell. + compute_announced: [AtomicU64; PRIORITIES], /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, submissions: AtomicU64, @@ -147,6 +152,8 @@ impl Port { render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), + compute_written: [const { AtomicU64::new(0) }; PRIORITIES], + compute_announced: [const { AtomicU64::new(0) }; PRIORITIES], effort_full: AtomicBool::new(false), submissions: AtomicU64::new(0), }), @@ -223,10 +230,35 @@ impl Port { unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; fence(Ordering::SeqCst); reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + self.compute_written[claim.priority].fetch_add(1, Ordering::AcqRel); claim.armed = false; Ok(next) } + /// Compute work slots written at `priority` so far. + pub(super) fn compute_written(&self, priority: u8) -> u64 { + self.compute_written + .get(usize::from(priority)) + .map_or(u64::MAX, |written| written.load(Ordering::Acquire)) + } + + /// Compute work slots at `priority` that a doorbell has announced. + pub(super) fn compute_announced(&self, priority: u8) -> u64 { + self.compute_announced + .get(usize::from(priority)) + .map_or(0, |announced| announced.load(Ordering::Acquire)) + } + + /// Rings the compute doorbell `message` of `priority`, after noting which slots it + /// announces. + pub(super) fn notify_compute(&self, priority: u8, message: u64) -> Result { + let index = usize::from(priority); + let written = self.compute_written.get(index).ok_or(EINVAL)?.load(Ordering::Acquire); + self.notify(message)?; + self.compute_announced[index].fetch_max(written, Ordering::AcqRel); + Ok(()) + } + /// Taken around every render publication; see `render`. pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { self.render.lock() diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5359a732562c4d..4b6bb8ce8c56f8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -594,15 +594,22 @@ pub(crate) struct Queue { source_readers: Arc, source_hint: crate::g17::sync_input::ReaderHint, /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more - /// unretired commands of this queue ahead of them, and the publication of the first. - withheld_doorbell: Option<(u64, u64)>, + /// unretired commands of this queue ahead of them, the publication of the first, and the + /// compute work slots written at its priority once its slot was. + withheld_doorbell: Option<(u64, u64, u64)>, } impl Queue { /// Returns the withheld doorbell once at most one unretired command of this queue - /// precedes the first publication it covers. - pub(crate) fn due_doorbell(&mut self) -> Option { - let (doorbell, first) = self.withheld_doorbell?; + /// precedes the first publication it covers. A doorbell of the same class rung after the + /// last withheld work slot was written announced every one of them: nothing is left to + /// ring. `announced` gives the slots announced per class. + pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { + let (doorbell, first, written) = self.withheld_doorbell?; + if announced(((doorbell >> 2) & 3) as u8) >= written { + self.withheld_doorbell = None; + return None; + } if self.active.iter().filter(|active| active.published < first).count() > 1 { return None; } @@ -1913,6 +1920,8 @@ pub(crate) trait Host { fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; + fn notify_compute(&mut self, priority: u8, message: u64) -> Result; + fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; } @@ -2260,12 +2269,14 @@ impl Queue { fence(Ordering::SeqCst); let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; if self.active.len >= WITHHOLD_DEPTH - && self.withheld_doorbell.is_none_or(|(withheld, _)| withheld == doorbell) + && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) { - self.withheld_doorbell.get_or_insert((doorbell, published)); + let written = host.compute_written(policy.priority()); + let entry = self.withheld_doorbell.get_or_insert((doorbell, published, written)); + entry.2 = written; } else { - host.notify(doorbell)?; - if self.withheld_doorbell.is_some_and(|(withheld, _)| withheld == doorbell) { + host.notify_compute(policy.priority(), doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _, _)| withheld == doorbell) { self.withheld_doorbell = None; } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 625339afafeaaf..90d980bc427c25 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1394,6 +1394,12 @@ impl compute::Host for ComputeHost<'_> { fn notify(&mut self, message: u64) -> Result { self.port.notify(message) } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } fn note_submission(&mut self) -> Result { self.effort.submit(); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) @@ -1438,6 +1444,12 @@ impl compute::Host for DirectHost<'_> { fn notify(&mut self, message: u64) -> Result { self.port.notify(message) } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } fn note_submission(&mut self) -> Result { Ok(()) } @@ -1680,13 +1692,13 @@ impl super::Firmware { freed |= busy && queue.channel_free(); // Withheld doorbells are rung once their command is next but one; one per // priority class announces every queue's work slots on that ring. - if let Some(doorbell) = queue.due_doorbell() { + if let Some(doorbell) = queue.due_doorbell(|class| port.compute_announced(class)) { let class = ((doorbell >> 2) & 3) as usize; doorbells[class] = doorbell; } } for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { - self.primary.notify(doorbell)?; + self.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. From 118c3b4c76a1768e2b7354d6e645b9de3198b978 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 13:42:20 -0400 Subject: [PATCH 56/68] drm/asahi: announce withheld G17 compute doorbells whenever retirement stalls A withheld compute doorbell waited for the host to retire all but one of the commands ahead of it. Retirement can stall even though the firmware has finished those commands, for example while a completion record waits up to 50 ms for its stamp. It can also leave ordinary operation through a timeout, kill, quarantine, replay, recovery witness or exit. The firmware then sat idle on a command it had never been told about. Count only predecessors that have no completion record yet. Ring a withheld doorbell at once from the compute scan, and from the timeout and cancellation paths, whenever the queue is settling. Never withhold while it is settling. The firmware then knows every published command, as it does without withholding. (cherry picked from commit d38cce2d8be54bdbc01e90e1a92ef3dd65ac0640) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 42 ++++++++++++++++--- .../drm/asahi/g17/runtime/compute_control.rs | 14 ++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4b6bb8ce8c56f8..9382d50d90b796 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -600,22 +600,53 @@ pub(crate) struct Queue { } impl Queue { - /// Returns the withheld doorbell once at most one unretired command of this queue - /// precedes the first publication it covers. A doorbell of the same class rung after the - /// last withheld work slot was written announced every one of them: nothing is left to - /// ring. `announced` gives the slots announced per class. + /// Returns the withheld doorbell once at most one command of this queue that precedes the + /// first publication it covers has no completion record yet. A doorbell of the same class + /// rung after the last withheld work slot was written announced every one of them: nothing + /// is left to ring. `announced` gives the slots announced per class. + /// + /// Retirement waiting for a stamp does not hold the doorbell back, and any departure from + /// ordinary retirement (timeout, kill, quarantine, replay, recovery or exit) rings it at + /// once: the firmware then sees every published command, as without withholding. pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { let (doorbell, first, written) = self.withheld_doorbell?; if announced(((doorbell >> 2) & 3) as u8) >= written { self.withheld_doorbell = None; return None; } - if self.active.iter().filter(|active| active.published < first).count() > 1 { + let settling = self.settling(); + let incomplete = self + .active + .iter() + .filter(|active| active.published < first && active.observed.is_none()) + .count(); + if !settling && incomplete > 1 { return None; } self.withheld_doorbell = None; Some(doorbell) } + + /// The queue has left ordinary retirement: a timeout, kill, quarantine, replay, recovery + /// witness or exit is in progress. + fn settling(&self) -> bool { + self.quarantined + || self.released + || self.exit.is_some() + || self.replays.len != 0 + || self.replay_retirement + || self.awaiting_witness + || self.retire_pending + || self.retired_by_teardown + || self.spared_quarantine + || self.failure_status_pending + || self.terminal.is_some() + } + + /// Takes the withheld doorbell regardless of its predecessors. + pub(crate) fn take_withheld_doorbell(&mut self) -> Option { + self.withheld_doorbell.take().map(|(doorbell, _, _)| doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( @@ -2269,6 +2300,7 @@ impl Queue { fence(Ordering::SeqCst); let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; if self.active.len >= WITHHOLD_DEPTH + && !self.settling() && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) { let written = host.compute_written(policy.priority()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed530..42282aea62023f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -69,7 +69,14 @@ impl Backend { if !queue.owns_packet(packet) { return Ok(None); } - queue.timeout(&mut |event| deferred.push(event)) + let status = queue.timeout(&mut |event| deferred.push(event)); + // A timed-out queue leaves ordinary retirement: announce what it withheld. + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } + status })(); if result.as_ref().is_ok_and(|status| status.is_some()) { // VM status and peer settlement precede signalling this timeout. The backend @@ -92,6 +99,11 @@ impl Backend { firmware.scan_compute(None, &mut deferred)?; if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } } Ok(()) })(); From a5f0863e49d31db23a986151fe9dee4b5fbbcb3e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 57/68] drm/asahi: build the G17 driver without warnings Kernels built with CONFIG_WERROR fail on the G17 changes. trace.c defines the eight tracepoint entry points that only Rust calls, so nothing in C declares them (-Wmissing-prototypes). The Rust side also leaves code without users: the compute host's generic doorbell, a recovery poll constant, the old capacity-wait helper and an unused import, and, without G16, the CPU timestamp mapping that only the G16 timestamp path uses. Declare the entry points in trace_api.h, included by trace.c, remove the unused items, and build the CPU timestamp mapping only with G16. Reported-by: RyanTheTide (cherry picked from commit 2d8286101c8631939cb13ce1b7b14c9041856ca6) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 - drivers/gpu/drm/asahi/g17/runtime.rs | 13 ------------ drivers/gpu/drm/asahi/g17/trace.c | 2 ++ drivers/gpu/drm/asahi/g17/trace.rs | 2 +- drivers/gpu/drm/asahi/g17/trace_api.h | 24 ++++++++++++++++++++++ drivers/gpu/drm/asahi/mmu.rs | 3 +++ drivers/gpu/drm/asahi/timestamp.rs | 1 + 7 files changed, 31 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/trace_api.h diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9382d50d90b796..9c4c5c92d049df 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1950,7 +1950,6 @@ pub(crate) trait Host { fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; - fn notify(&mut self, message: u64) -> Result; fn notify_compute(&mut self, priority: u8, message: u64) -> Result; fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 90d980bc427c25..1600c55eaf47b5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -31,7 +31,6 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; /// Pending admission keeps exactly retired compute backing available for reuse. @@ -1391,9 +1390,6 @@ impl compute::Host for ComputeHost<'_> { fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } fn notify_compute(&mut self, priority: u8, message: u64) -> Result { self.port.notify_compute(priority, message) } @@ -1441,9 +1437,6 @@ impl compute::Host for DirectHost<'_> { fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } fn notify_compute(&mut self, priority: u8, message: u64) -> Result { self.port.notify_compute(priority, message) } @@ -2396,12 +2389,6 @@ impl Registry { } impl Backend { - fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - self.reclaim_capacity(admission, reclaim_backing)?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - admission.check() - } - /// Compute admission without a channel: give back what pressure allows, then block until a /// channel is released after `observed` (sampled before the failed check). fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index ae977c13aab3bd..23666be389b969 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -2,6 +2,8 @@ #include #include +#include "trace_api.h" + #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index 57db5abaccef86..b76b46425d2ce9 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -3,7 +3,7 @@ //! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a //! static-key test in C while its event is disabled. -use kernel::{bindings, dma_fence::RawDmaFence, prelude::*, sync::Arc}; +use kernel::{bindings, dma_fence::RawDmaFence, sync::Arc}; unsafe extern "C" { fn asahi_g17_trace_publish_enabled() -> bool; diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h new file mode 100644 index 00000000000000..d09f529a163f0f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Entry points of the G17 command lifecycle tracepoints, called from Rust + * (g17/trace.rs). + */ +#ifndef _ASAHI_G17_TRACE_API_H +#define _ASAHI_G17_TRACE_API_H + +#include + +struct dma_fence; + +bool asahi_g17_trace_publish_enabled(void); +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence); +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end); +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind); +bool asahi_g17_trace_doorbell_enabled(void); +void asahi_g17_trace_doorbell(u8 engine, u32 power); +void asahi_g17_trace_message(u8 kind, u64 value); +void asahi_g17_trace_event(u8 phase); + +#endif /* _ASAHI_G17_TRACE_API_H */ diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7e8945638438ad..5f54c9773cb33a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1283,11 +1283,13 @@ impl TimestampVmap { /// Prepared CPU timestamp access with its own vmap; destruction happens after completion /// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) struct TimestampMapping { mapping: TimestampVmap, offset: usize, } +#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { /// Prepare one bounded CPU timestamp destination on retained GEM backing. pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { @@ -1340,6 +1342,7 @@ impl KernelMapping { /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { let gem = self.0._gem.as_ref().ok_or(EINVAL)?; TimestampMapping::new(gem, self.timestamp_offset(offset)?) diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 0d365a8d930bbf..9f13436b4caf55 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -66,6 +66,7 @@ impl Buffer { } } + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), From 870e51b01269b0580cd8e913d61a68bb40da38af Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 58/68] drm/asahi: renew G17 job deadlines while the GPU keeps retiring work The scheduler fails a published job two seconds after it started unless the job is a ray-tracing or acceleration-structure build. Under load that timer measures how the GPU is shared rather than whether the job makes progress. Beside four other clients, a 1.5 s compute dispatch took up to 5 s, and lower-priority work waited behind higher-priority work for seconds, while the GPU kept retiring other commands at least every 1.2 s. Those jobs failed with a lost device although nothing had hung. Renew a published job's deadline while any command on the GPU retired in the last five seconds, at most fourteen times (thirty seconds in all). A GPU that retires nothing for five seconds has stopped, and its jobs fail as before. (cherry picked from commit 9a0ed807e4d3691a5ea46fce2a2424295f8af6ea) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/job.rs | 9 ++++ drivers/gpu/drm/asahi/g17/recovery.rs | 2 +- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 11 +++++ drivers/gpu/drm/asahi/g17/timeout.rs | 44 ++++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index ea78b799085fb9..647b20e9753219 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -307,6 +307,7 @@ struct Timeout { backend: Arc, victim_renewals: u8, execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -331,6 +332,7 @@ impl sched::JobImpl for Job { backend: self.backend.clone(), victim_renewals: 0, execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -344,6 +346,13 @@ impl sched::JobImpl for Job { ) { return true; } + // Other work retiring means the GPU is shared, not hung. + if super::timeout::renew_shared( + job.packet.is_published(), super::recovery::since_retired(super::now_ns()), + &mut job.shared_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred { job.victim_renewals = job.victim_renewals.saturating_add(1); diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 84a01d975c36d5..36f1ceac9f821c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,7 +10,7 @@ mod attribution; mod slot; mod safety; -pub(crate) use safety::note_retired; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index b97fc269143290..082416f0cb9a66 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -10,16 +10,27 @@ const LIMIT: u8 = 8; /// Commands retired successfully. A watchdog storm means the GPU stopped making progress; /// timeouts of overloaded clients while other work keeps retiring are not one. static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); /// Called for every command that retires without error. pub(crate) fn note_retired() { RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); } pub(super) fn retired() -> u64 { RETIRED.load(Ordering::Relaxed) } +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + pub(super) struct Storm { start: Option, count: u8, diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index d951bd164d6e96..7b4ef4137b7958 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -53,6 +53,32 @@ pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) true } +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Renews a published job's deadline while some command retired within +/// [`LIVENESS_NS`] (`since_retired`), at most [`SHARED_RENEWALS`] times. +pub(crate) fn renew_shared(published: bool, since_retired: Option, renewals: &mut u8) -> bool { + if !published + || !since_retired.is_some_and(|since| since < LIVENESS_NS) + || *renewals >= SHARED_RENEWALS + { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. @@ -215,6 +241,24 @@ mod tests { assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); } #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, Some(4_900_000_000), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, Some(0), &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, Some(LIVENESS_NS), &mut renewals)); + assert!(!renew_shared(true, None, &mut renewals)); + assert!(!renew_shared(false, Some(0), &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; From 6fafec6c87ea3b5ba1b5f5c48fcc2bece4ee9d6c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 59/68] drm/asahi: kill closed G17 contexts only while no other work runs A client whose work failed closes its queues, and 250 ms later the driver asks the firmware to kill the closed context's remaining commands. While other contexts' commands are running, that kill makes the firmware restart the GPU and fail everything in flight, and a few repeats crash it. On an otherwise idle GPU the same kill is handled cleanly. Publish the kill only while no other context has a render pair or compute queue in flight, or once the GPU has retired nothing for five seconds, the watchdog's liveness window. Until then the closed context's queues stay quarantined and retained, and its commands retire like any others once the GPU reaches them, as commands with ray traversal scratch already do. Deferred kills are rechecked on every service pass. (cherry picked from commit afbfbc0b89ca4779b13cefff5bed0b4036751b2c) Signed-off-by: DjDeveloperr --- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 26 ++++++++++++++++++- .../drm/asahi/g17/runtime/render/teardown.rs | 8 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 69bc7550fc4e92..bf9d023038a51b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,6 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, + timeout::LIVENESS_NS, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, Ordering}; @@ -82,7 +83,7 @@ impl Firmware { }) else { return Ok(()); }; - if self.render_control_backpressured() { + if self.render_control_backpressured() || self.kill_unsafe(context, now) { return Ok(()); } let (owner, target) = { @@ -149,6 +150,29 @@ impl Firmware { Ok(()) } + /// A kill while other contexts' commands run makes the firmware restart the + /// GPU, and repeated ones crash it; on an otherwise idle GPU it is safe. A + /// closed context's commands left alone retire like any others once the GPU + /// reaches them, so its queues stay quarantined and retained until then. Only + /// a GPU that retired nothing for [`LIVENESS_NS`] has stopped, and then the + /// kill proceeds. + fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); + if stalled { + return false; + } + self.queues.render.foreign_in_flight(context) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + queue.in_flight() && !queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + }) + } + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { if self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index eb04f803619558..e54b1f4638f587 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -46,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self From 4ef3920ec9d36c9a19098b0099f8fb6c7064550d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 18:22:31 -0400 Subject: [PATCH 60/68] drm/asahi: fail G17 render groups a recovery dropped out of order at once After a firmware restart, the firmware retires every render group it had started on a pair without running it: the groups the recovery table lists, and those whose tiler had run while their fragment stage was still pending. Groups it had not started run as usual. The host republishes dropped groups only when every retained group was dropped, so that none runs after a live successor. Once a later group has run, the dropped ones never qualify, and in-order retirement cannot pass the unexecuted front either. The pair then waited for the render watchdog: about 2 s later before deadline renewal, up to 30 s with it while other work keeps retiring. Its owner then saw ETIMEDOUT and a poisoned VM, even when the client was innocent and none of its work had started. In a stress run with forced recoveries, 10 of 11 recoveries left such a pair. Replay was also wrong for a dropped group that a compute command waited on: the wait ends when firmware retires the group, so the consumer may read output the group never wrote while the replay runs it afterwards. A client in the same stress run read back the previous iteration's image this way without any error. When firmware has retired every group of the pair but a dropped group cannot run again in order (a later group already ran, the dropped group had started, or a compute command waited on it), fail the whole set at once, after the same double observation as a replay. The groups that ran fail too: they consumed output the dropped ones never wrote. The work is innocent, so it fails with ENODATA, which its owner sees, without poisoning the VM. Guilty and blamed sets are left to the timeout as before, and a group that ran ahead of every dropped one is still retired in order. (cherry picked from commit 31279abf0648c00e454b181f01d65b277a32fb90) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/queue/render.rs | 79 ++++++++++++++++--- .../asahi/g17/queue/render/dependencies.rs | 26 +++--- drivers/gpu/drm/asahi/g17/replay.rs | 67 +++++++++++++++- .../asahi/g17/runtime/render/dependencies.rs | 3 + 4 files changed, 149 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 9469d526f49c8e..696a1486eb0ec5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1333,10 +1333,9 @@ impl Pair { let blamed = active.packet.completion.blamed(); let exhausted = active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is not abandoned, an error included. - let abandoned = innocent - && !blamed - && (|| -> Result { + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { let mut observer = Observer { memory, qids, @@ -1347,21 +1346,33 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. - // Interrupted work is run again only if it never started. - Ok(matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) - ) && (!active.packet.completion.interrupted() || snapshot.untouched())) + let witness = |executed| { + matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) })() .unwrap_or_else(|error: Error| { failure = Some(error); - false - }); + (false, false, false) + }) + } else { + (false, false, false) + }; Group { innocent, blamed, abandoned, exhausted, + ran, + lost, } }); let decision = replay::decide(groups); @@ -1376,7 +1387,7 @@ impl Pair { front.abandoned_since = None; return Ok(false); } - Decision::Replay => {} + Decision::Replay | Decision::Fail => {} } if grace { let Some(since) = front.abandoned_since else { @@ -1387,6 +1398,9 @@ impl Pair { return Ok(false); } } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } // decide() lets only the front be exhausted. An exhausted front fails, and its // successors, which consume its output, fail with it rather than run without it. let mut failing = self.active.front().is_some_and(|front| { @@ -1445,6 +1459,45 @@ impl Pair { Ok(true) } + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + Ok(true) + } + /// A front group a restart interrupted after its tiler started, which firmware /// then retired without writing its fragment end, may have run partly. It is /// not replayed (blending or storage writes could apply twice) and fails now, @@ -1943,7 +1996,7 @@ impl Pair { pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { let mut dependencies = RenderDependencies::EMPTY; let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier)?; + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; for dependency in frontier.as_slice() { *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index 53a0fa6e449ed7..ab96b48de3e355 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -31,6 +31,7 @@ fn include( prefix: Option, packet: &Arc, word: u64, + compute: bool, ) -> Result { if !Arc::ptr_eq(packet.completion.status(), status) { return Ok(()); @@ -38,12 +39,15 @@ fn include( if !job::Order::contains(prefix, packet.order.sequence) { return Ok(()); } - frontier.include( - prefix, - packet.order.sequence, - job::fence_status(&packet.completion.scheduler_fence()), - word, - ) + let status = job::fence_status(&packet.completion.scheduler_fence()); + frontier.include(prefix, packet.order.sequence, status, word)?; + // A compute consumer's firmware wait ends once firmware retires this group, even unexecuted + // after a restart, so it may then read output the group never wrote. Render consumers of the + // same owner follow the group on its pair, and a replay keeps that order. + if compute && status == 0 { + packet.completion.note_firmware_consumer(); + } + Ok(()) } impl Dependencies { @@ -60,12 +64,13 @@ impl Dependencies { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.word)?; + include(frontier, status, prefix, &entry.packet, entry.word, compute)?; } Ok(()) } @@ -80,7 +85,7 @@ impl Dependencies { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + include(frontier, status, prefix, &entry.packet, entry.ta_word, false)?; } Ok(()) } @@ -112,6 +117,7 @@ impl Pair { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); @@ -120,7 +126,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, compute)?; } Ok(()) } @@ -138,7 +144,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, false)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs index 94daa5ec0dfebc..7668871376bad2 100644 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -16,6 +16,12 @@ pub(crate) struct Group { pub(crate) abandoned: bool, /// Already republished the maximum number of times. pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -25,26 +31,54 @@ pub(crate) enum Decision { Wait, /// Never replay this set; the render timeout settles it. Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, } /// Groups are read in order and the first one that decides stops the walk, so a caller can /// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its /// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. pub(crate) fn decide(groups: impl Iterator) -> Decision { let mut front_exhausted = None; let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; for group in groups { if !group.innocent || group.blamed { return Decision::Refuse; } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } if !group.abandoned { return Decision::Wait; } + dropped = true; match front_exhausted { None => front_exhausted = Some(group.exhausted), Some(_) => successor_exhausted |= group.exhausted, } } + if unordered { + return Decision::Fail; + } match front_exhausted { Some(true) => Decision::Replay, Some(false) if !successor_exhausted => Decision::Replay, @@ -56,10 +90,13 @@ pub(crate) fn decide(groups: impl Iterator) -> Decision { mod tests { use super::*; fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted } + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } } - const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; - const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; #[test] fn every_group_abandoned_and_innocent_is_replayed() { assert_eq!(decide([OK].into_iter()), Decision::Replay); @@ -106,4 +143,28 @@ mod tests { Decision::Replay ); } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 2b14b4548e0db8..e150a10e2c9bc7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -12,6 +12,7 @@ impl Registry { packet: &Arc, ) -> Result { let mut frontier = Frontier::new(); + let compute = packet.engine() == crate::g17::job::Engine::Compute; for entry in self.render.entries.iter().flatten() { if let Some(pair) = entry.pair.as_ref() { pair.collect_dependencies( @@ -19,6 +20,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { dependencies.collect( @@ -26,6 +28,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } } From 045f59cb4bd987f1a59a468ed7cc1a12a35d36a7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 18:22:31 -0400 Subject: [PATCH 61/68] drm/asahi: announce withheld G17 compute doorbells after a recovery A compute doorbell is withheld while two or more unretired commands of its queue precede it, and any departure from ordinary retirement is meant to ring it at once. A recovery that touches no command of the queue left it withheld until those predecessors retired. Once the firmware has resumed from a restart, ring every withheld doorbell, one per priority class, before any replay is published. (cherry picked from commit 7012a59ed129baf969d83a3aa4d90f2a6cb410f2) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/recovery.rs | 4 ++++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 36f1ceac9f821c..3bb9d20190b57b 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -107,6 +107,9 @@ pub(super) trait Queues { fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; fn release_retired_compute(&mut self); + /// Ring every compute doorbell still withheld: a restart is a departure from ordinary + /// retirement, after which the firmware must see every published command. + fn announce_withheld(&mut self); } /// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions @@ -296,6 +299,7 @@ pub(super) fn service( } let scan_error = host.rescan_compute().err(); host.release_retired_compute(); + host.announce_withheld(); Ok(ServiceResult { scan_error, target_error }) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e13ec3e35e13cc..4cf2dd0fafb970 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -236,6 +236,26 @@ impl recovery::Queues for Service<'_> { let result = self.firmware.release_compute_witnesses(self.deferred); self.settlement_error = self.settlement_error.or(result.err()); } + fn announce_withheld(&mut self) { + let mut doorbells = [0u64; 4]; + for mut queue in self + .firmware + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if let Some(doorbell) = queue.take_withheld_doorbell() { + doorbells[((doorbell >> 2) & 3) as usize] = doorbell; + } + } + // One doorbell per priority class announces every queue's work slots on that ring. + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + let result = self.firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell); + self.settlement_error = self.settlement_error.or(result.err()); + } + } } impl Firmware { From b8b841ffd507cc77fabc39b23a8331b15b37e3e4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 19:17:20 -0400 Subject: [PATCH 62/68] drm/asahi: move the G17 cumulative-signals feature to bit 29 Feature bit 17 is also used for render resources by the M5 userspace line, whose features take bits 17 to 28. Advertise cumulative signals as bit 29, which no line uses, so one Mesa can serve both kernels. A kernel and Mesa that disagree on the bit fall back to ordinary per-point signals, so either mix still works. (cherry picked from commit 8e257af33a6c4036ee56bc8a628e733dba2bd525) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/sync_output.rs | 2 +- include/uapi/drm/asahi_drm.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs index 50585befb3b71f..928ebe23193948 100644 --- a/drivers/gpu/drm/asahi/g17/sync_output.rs +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -6,7 +6,7 @@ use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; // Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these // local permits building this module against an older kernel's Rust bindings. pub(crate) const SYNC_CUMULATIVE: u32 = 2; -pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 17; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 29; unsafe extern "C" { fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 836739ffbbebfb..66d51881993838 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -374,7 +374,7 @@ enum drm_asahi_feature { * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs * include the earlier points of that timeline without adding GPU waits. */ - DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 17, + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 From 2ca07792259ebc0014fc16b4db0dd384f91a4523 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:29:35 -0400 Subject: [PATCH 63/68] drm/asahi: build the G17 compute QID budget test against the variant constructor The test that grows compute QIDs with free identities still called Registry::new() with the clusters and descriptor flags only, from before the constructor took the deflake setting and the firmware variant, so a KUnit build of the driver failed. Construct the registry as the other tests do, for one cluster on the 26A428 firmware. (cherry picked from commit 76d4e880149e5592396001f728f7803efb616377) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 1600c55eaf47b5..c5675cbec5f797 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2547,7 +2547,7 @@ mod shared_capacity_tests { #[test] fn compute_grows_with_free_qids_and_narrows_queue_admission() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); for _ in 0..20 { slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); From ee16ba07a46ae16f46d0ccace15f5e8373c9dd0a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:35:37 -0400 Subject: [PATCH 64/68] drm/asahi: check the G17 QID floor test against the open-queue compute budget The test that a firmware QID floor admits only the render pairs it leaves reserved every compute channel before opening any logical queue. Compute QIDs are now sized by the open logical queues, so on 25G83's floor that first reservation was refused and the test panicked. Open the 42 logical queues the floor allows without the up-front compute reservations, and check as before that the next one is refused when it is created. (cherry picked from commit d0da7e79dc9c99020d109e95d18256cd5b3d6bc2) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/runtime.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index c5675cbec5f797..e04295d5766063 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2508,9 +2508,6 @@ mod shared_capacity_tests { GFP_KERNEL, ) .unwrap(); - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { - registry.reserve_compute(owner).unwrap(); - } let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { From 578dd9535249646667d34bd8387ea7c8082e4ca1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:58:53 -0400 Subject: [PATCH 65/68] drm/asahi: check the G17 admission churn test against its deadline, not the clock The test that gate churn cannot renew a compute admission budget looped until the budget ran out and then required the whole loop to have taken less than 500 ms. The loop does not sleep, so that bound only measured how long the test thread was descheduled, and a KUnit run under KVM on a shared CI host exceeded it. Check the property instead: every admission granted came from an attempt that started before the deadline, and the first attempt after it is refused with EBUSY. (cherry picked from commit 18b408b7368b9470c22d92c9afe1991d0fe608ba) Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/g17/preparation.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 3cafecf5d82067..eb2366fa3451b2 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -361,9 +361,9 @@ mod compute_deadline_tests { #[test] fn recovery_and_binding_retry_churn_cannot_renew_budget() { let gate = Gate::new().unwrap(); - let start = super::super::now_ns(); + let deadline = super::super::now_ns() + 20_000_000; let check = || Ok(()); - let admission = Admission::new(start + 20_000_000, &check); + let admission = Admission::new(deadline, &check); loop { // Re-entry models a channel taken between admission/publication. // Recovery joins each dropped lease, then wakes the same budget. @@ -371,12 +371,17 @@ mod compute_deadline_tests { gate.changed.notify_all(); gate.wait_drained(); assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); match gate.enter_until(&admission) { - Ok(lease) => drop(lease), + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } Err(error) => { assert_eq!(error, EBUSY); break; } } } - assert!(super::super::now_ns() - start < 500_000_000); assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); } From 18c64047548c55b8122368a4410153e1a4107d7d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 08:00:01 -0400 Subject: [PATCH 66/68] drm/asahi: m3: admit clients that ask for the parameters up to the HAL generation The M3 runtime admits a VM only for a file that asked GET_PARAMS for at least AGX3_PARAMS_SIZE bytes, and that was the size of drm_asahi_params_global. The G17 update grows the structure from 600 to 616 bytes with two optional tail fields, memory_limit and client_memory_limit, so Mesa built against the 600-byte structure would be refused VM creation with EOPNOTSUPP until it is rebuilt. The check only needs to know that the client knows the AGX3 fields. Pin the minimum at the end of gpu_hal_generation, 600 bytes, and assert that value at build time. GET_PARAMS still copies only the size the client asked for. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/m3/m3_client.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 8fc7791e939ba1..dff1af0783965d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -5,11 +5,11 @@ //! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for //! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 //! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client -//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at -//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks -//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore -//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a -//! VM, so a refused client can still query the parameters but never reaches the hardware. +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields +//! of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks GET_PARAMS +//! for at least the structure up to them; older userspace asks for less. On G15 a file therefore +//! has to ask for those fields before it can create a VM. Every path to the GPU needs a VM, so a +//! refused client can still query the parameters but never reaches the hardware. use core::sync::atomic::{ AtomicU32, @@ -29,15 +29,20 @@ use crate::{ mmu, }; -/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. -const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; +/// GET_PARAMS size an AGX3-aware client asks for: the global parameter structure up to the end +/// of `gpu_hal_generation`. The optional tail fields that follow it are not part of the check, +/// so growing the structure does not refuse userspace built against the shorter one. +const AGX3_PARAMS_SIZE: u64 = + (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + + core::mem::size_of::()) as u64; +const _: () = assert!(AGX3_PARAMS_SIZE == 600); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; static REFUSALS: AtomicU32 = AtomicU32::new(0); -/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the -/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for at least +/// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { { From 1032928116c13840efaf0ba6ae68fee4e1cd70ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 08:00:55 -0400 Subject: [PATCH 67/68] drm/asahi: apply the GPU memory limits and RSS attribution to G17 only The main probe configured the GPU memory ledger for every GPU, so the new defaults (mem_limit_pct=67, client_mem_limit_pct=50, oom_attribution=1) also applied to M1, M2 and the M3 runtime. Those GPUs would refuse BO creation with ENOMEM once committed sizes passed the fractions, even with nothing resident, and would count GPU memory in the creator's RSS for the OOM killer. The separate G15 probe never configured the limits, but still got the RSS attribution, because it was read from the module parameter on every object. That is a policy change for every Mac, and it should be proposed on its own. Until then, configure the ledger in the G17 probe only, and make configure() also decide the RSS attribution. Every other GPU keeps the totals, in fdinfo and sysfs, with no limit and no attribution, and reports zero (no driver limit) in the GET_PARAMS tail. The module parameter descriptions now say they apply to G17. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/asahi_core.rs | 6 +++--- drivers/gpu/drm/asahi/driver.rs | 3 ++- drivers/gpu/drm/asahi/memacct.rs | 26 ++++++++++++++++++-------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 8b782a6f8ffa74..84a9bff28d6d75 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -176,15 +176,15 @@ module! { }, mem_limit_pct: u32 { default: 67, - description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", + description: "G17: device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", }, client_mem_limit_pct: u32 { default: 50, - description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", + description: "G17: per-client GPU memory limit in percent of RAM (0: the device limit)", }, oom_attribution: u8 { default: 1, - description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", + description: "G17: count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 9b9b8fc4ccd152..7b2c2b1be0f2f6 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -236,7 +236,6 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); - crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -394,6 +393,8 @@ impl AsahiPlatformDriver { pdev: &'bound platform::Device>, cfg: &'static g17::Config, ) -> Result> { + // The GPU memory limits and RSS attribution apply only to G17 for now. + crate::memacct::LEDGER.configure(); // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index 9da09968171e4a..ac26439d8a3f96 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -30,6 +30,10 @@ //! Imported dma-bufs are charged to their exporter and are not counted here. //! Kernel objects (firmware structures, per-queue buffers and the global heaps) //! count towards the device total and its limit, but are attributed to no process. +//! +//! The limits and the RSS attribution apply only to G17 for now: only its probe +//! calls [`Ledger::configure`]. Other GPUs keep the totals, with no limit and no +//! attribution. use core::sync::atomic::{ AtomicBool, @@ -78,6 +82,9 @@ const _: () = { /// The ledger of the (single) AGX GPU. pub(crate) static LEDGER: Ledger = Ledger::new(); +/// Whether held user objects are charged to their creator's RSS. Set by [`Ledger::configure`]. +static ATTRIBUTE_RSS: AtomicBool = AtomicBool::new(false); + /// The limits that follow from the amount of RAM and two percentages. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Limits { @@ -148,7 +155,7 @@ impl Ledger { } } - /// Sets the limits from the module parameters and the amount of RAM. + /// Sets the limits and the RSS attribution from the module parameters and the amount of RAM. pub(crate) fn configure(&self) { // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; @@ -161,6 +168,10 @@ impl Ledger { *module_parameters::client_mem_limit_pct.value(), ); self.set_limits(limits); + ATTRIBUTE_RSS.store( + *module_parameters::oom_attribution.value() != 0, + Ordering::Relaxed, + ); pr_info!( "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", limits.device >> 20, @@ -300,11 +311,6 @@ impl Charge { /// A user object admitted by `commit`, attributed to `mm` if there is one. pub(crate) fn user(commit: Commit, mm: Option>) -> Self { let bytes = commit.bytes; - let mm = if *module_parameters::oom_attribution.value() != 0 { - mm - } else { - None - }; Self { kind: Kind::User { commit, mm }, bytes, @@ -312,9 +318,13 @@ impl Charge { } } - /// A user object attributed to the current process. + /// A user object attributed to the current process, if the ledger attributes RSS. pub(crate) fn user_current(commit: Commit) -> Self { - let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + let mm = if ATTRIBUTE_RSS.load(Ordering::Relaxed) { + kernel::current!().mm().map(|mm| ARef::from(&**mm)) + } else { + None + }; Self::user(commit, mm) } From 6c66b41af3e77c0509ce24f3615c1bc2016cea14 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 21:44:10 -0400 Subject: [PATCH 68/68] drm/asahi: restore the DRM driver's module owner The G17 sync (4b7aa809ff54) took drivers/gpu/drm/asahi/driver.rs from aurora-silicon, whose Rust DRM layer derives the file_operations owner from the driver's OwnerModule type. This tree's layer still takes it from drm::Driver::MODULE, which defaults to None, so the sync dropped the AsahiDrmDriver override that set it to THIS_MODULE. With DRM_ASAHI=m, the GEM file_operations and the dma-bufs that drm_gem_prime_export() creates then have a NULL owner. Open DRM files and exported buffers no longer hold a reference on the module, so it could be unloaded while their callbacks still point into it. Restore the override, and assert at build time that MODULE is set, so a later sync that drops it fails to compile. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/driver.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 7b2c2b1be0f2f6..d93657e677553d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -91,6 +91,10 @@ const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { desc: c"Apple AGX Graphics", }; +// Open DRM files and exported DMA buffers take their callbacks' module from `MODULE`. With +// `None`, a modular build could be unloaded while they still point into it. +const _: () = assert!(::MODULE.is_some()); + /// DRM Driver implementation for `AsahiDriver`. #[vtable] impl drm::driver::Driver for AsahiDrmDriver { @@ -103,6 +107,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true;