diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c index 5e10e8df372f13..64abdcc4305801 100644 --- a/drivers/dma-buf/dma-fence-array.c +++ b/drivers/dma-buf/dma-fence-array.c @@ -107,17 +107,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) unsigned int i; /* - * We need to read num_pending before checking the enable_signal bit - * to avoid racing with the enable_signaling() implementation, which - * might decrement the counter, and cause a partial check. - * atomic_read_acquire() pairs with atomic_dec_and_test() in - * dma_fence_array_enable_signaling() - * - * The !--num_pending check is here to account for the any_signaled case - * if we race with enable_signaling(), that means the !num_pending check - * in the is_signalling_enabled branch might be outdated (num_pending - * might have been decremented), but that's fine. The user will get the - * right value when testing again later. + * Read the counter before the enable bit so callback installation + * cannot make a partial count look complete. The acquire pairs with + * the counter decrements in dma_fence_array_enable_signaling(). + * If enabling races with the lazy scan, every child must still be + * observed complete before that scan can return true. */ num_pending = atomic_read_acquire(&array->num_pending); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) { @@ -126,8 +120,16 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) return false; } + /* + * Arrays require every member to complete. A pending member already + * proves the array incomplete; polling its successors only repeats work + * during timeline-chain walks. Keep the completion counter and the + * enabled-signaling path unchanged. + */ for (i = 0; i < array->num_fences; ++i) { - if (dma_fence_is_signaled(array->fences[i]) && !--num_pending) + if (!dma_fence_is_signaled(array->fences[i])) + return false; + if (!--num_pending) goto signal; } return false; @@ -256,8 +258,7 @@ EXPORT_SYMBOL(dma_fence_array_init); * and fill it with the fences it wants to add to the object. Ownership of this * array is taken and dma_fence_put() is used on each fence on release. * - * If @signal_on_any is true the fence array signals if any fence in the array - * signals, otherwise it signals when all fences in the array signal. + * The array signals when every fence in the array signals. */ struct dma_fence_array *dma_fence_array_create(int num_fences, struct dma_fence **fences, diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index 4e7ee25372ba6f..45a50cef83cc93 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -15,6 +15,7 @@ struct mock_fence { struct dma_fence base; spinlock_t lock; + atomic_t polls; }; static const char *mock_name(struct dma_fence *f) @@ -22,9 +23,18 @@ static const char *mock_name(struct dma_fence *f) return "mock"; } +static bool mock_signaled(struct dma_fence *fence) +{ + struct mock_fence *f = container_of(fence, struct mock_fence, base); + + atomic_inc(&f->polls); + return false; +} + static const struct dma_fence_ops mock_ops = { .get_driver_name = mock_name, .get_timeline_name = mock_name, + .signaled = mock_signaled, }; static struct dma_fence *__mock_fence(u64 context, u64 seqno) @@ -36,6 +46,7 @@ static struct dma_fence *__mock_fence(u64 context, u64 seqno) return NULL; spin_lock_init(&f->lock); + atomic_set(&f->polls, 0); dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno); return &f->base; @@ -584,7 +595,71 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test) dma_fence_put(f1); } +/* + * A pending all-of array needs no polls of later members. Keep its + * independent member fences so their polling and out-of-order completion + * remain observable without enabling callbacks on the array. + */ +static void test_array_pending_prefix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + struct mock_fence *last_mock; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + last_mock = container_of(last, struct mock_fence, base); + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + KUNIT_EXPECT_EQ(test, atomic_read(&last_mock->polls), 0); + dma_fence_signal(last); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(first); + KUNIT_EXPECT_TRUE(test, dma_fence_is_signaled(array)); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + +static void test_array_pending_suffix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + dma_fence_signal(first); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + /* Installing callbacks after a lazy partial poll must still account + * for the already signaled first member and wait for the last. + */ + dma_fence_enable_signaling(array); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(last); + KUNIT_EXPECT_GT(test, dma_fence_wait_timeout(array, false, HZ), 0L); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + static struct kunit_case dma_fence_unwrap_cases[] = { + KUNIT_CASE(test_array_pending_prefix), + KUNIT_CASE(test_array_pending_suffix), KUNIT_CASE(test_sanitycheck), KUNIT_CASE(test_unwrap_array), KUNIT_CASE(test_unwrap_chain), diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 6a5783c4c54da8..037ffb71a9d4cf 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o +CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o asahi-$(CONFIG_DRM_ASAHI_UAT_ADOPTION) += g15/uat_glue.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0fbe6e728773d3..84a9bff28d6d75 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -25,6 +25,7 @@ mod gpu; mod hw; mod initdata; mod mem; +mod memacct; mod m3; mod microseq; mod mmu; @@ -35,6 +36,7 @@ mod regs; mod slotalloc; mod stats; mod sysfs_exports; +mod timestamp; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -172,5 +174,17 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, + mem_limit_pct: u32 { + default: 67, + description: "G17: device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", + }, + client_mem_limit_pct: u32 { + default: 50, + description: "G17: per-client GPU memory limit in percent of RAM (0: the device limit)", + }, + oom_attribution: u8 { + default: 1, + description: "G17: count held GPU memory in its creator's RSS for the OOM killer (1 = on)", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d8..d93657e677553d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -91,6 +91,10 @@ const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { desc: c"Apple AGX Graphics", }; +// Open DRM files and exported DMA buffers take their callbacks' module from `MODULE`. With +// `None`, a modular build could be unloaded while they still point into it. +const _: () = assert!(::MODULE.is_some()); + /// DRM Driver implementation for `AsahiDriver`. #[vtable] impl drm::driver::Driver for AsahiDrmDriver { @@ -153,6 +157,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -390,6 +398,8 @@ impl AsahiPlatformDriver { pdev: &'bound platform::Device>, cfg: &'static g17::Config, ) -> Result> { + // The GPU memory limits and RSS attribution apply only to G17 for now. + crate::memacct::LEDGER.configure(); // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -419,6 +429,11 @@ impl AsahiPlatformDriver { return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + } Ok(AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, @@ -433,7 +448,7 @@ impl AsahiPlatformDriver { impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a5..0c4c6048408262 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + memacct, + gpu, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -122,10 +127,18 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, + bundled: bool) -> Result + { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -146,6 +159,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -168,12 +183,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -188,12 +201,14 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -209,7 +224,25 @@ impl SyncItem { // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push(Self::parse_one(file, sync, i >= in_count, + allow_cumulative)?, GFP_KERNEL)?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], anchor - in_count as usize)?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -218,7 +251,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -236,6 +269,8 @@ pub(crate) struct File { /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, + /// GPU memory this client created and holds. + mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -255,9 +290,20 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } @@ -270,26 +316,37 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + crate::util::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + crate::util::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + crate::util::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); + addr_of_mut!((*slot).mem).write(mem); Ok(()) }) } @@ -341,7 +398,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +417,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -383,7 +443,7 @@ impl File { // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { @@ -402,15 +462,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +484,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +555,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +646,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -589,7 +669,7 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); + reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -632,8 +712,24 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = gem::user_pages_no_reclaim( + size, + reg_data.gpu.as_any().is::(), + ); + let gem = gem::new_object( + device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + )?; if reg_data.gpu.prepare_new_objects() { gem::prepare_new_object(&gem)?; } @@ -713,11 +809,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +830,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +848,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1296,27 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + return Err(EINVAL); + } + // SAFETY: capable() only reads the calling task's credentials (ioctl process context). + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + && !file.is_current_master() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1260,6 +1378,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1413,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1305,7 +1426,8 @@ impl File { copy_commands(&mut vec)?; } let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, + gpu.as_any().is::())?; mod_dev_dbg!( device, @@ -1369,7 +1491,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 78b5058c8d33f7..e83ffe05934d3d 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -365,7 +365,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 8bfa34a5892fde..5228c145bb87cc 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -650,7 +650,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_destroyed(&self) { vm::closed(self) } @@ -676,7 +676,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index f457b924cbb943..0f0aa01e235177 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -101,7 +101,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs new file mode 100644 index 00000000000000..8014266f0b0ab1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resource-specific scheduler waits. A caller samples a domain before a failed try, then +//! registers against that sample after rolling back unpublished claims; the release that frees +//! the resource bumps the generation and signals exactly the waiters of its own domain. +//! Successful acquisition never allocates a fence, and a release without waiters costs one +//! lock round trip. + +use super::fence::{self, CompletionFence}; +use kernel::{ + c_str, + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +struct Entry { + fence: UserFence, + next: Option>, +} + +struct State { + generation: u64, + waiters: Option>, +} + +#[pin_data] +pub(crate) struct Availability { + #[pin] + state: Mutex, + fences: FenceContexts, +} + +impl Availability { + pub(crate) fn new() -> Result> { + Arc::pin_init( + try_pin_init!(Self { + state <- new_mutex!( + State { generation: 0, waiters: None }, + "g17::Availability" + ), + fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + }), + GFP_KERNEL, + ) + } + + /// Sample before checking capacity under the resource's own synchronization. + pub(crate) fn observe(&self) -> u64 { + self.state.lock().generation + } + + pub(crate) fn has_waiters(&self) -> bool { + self.state.lock().waiters.is_some() + } + + /// A fence the next notification signals. None means capacity changed after the sample: + /// retry instead of waiting. Called with no resource lock held. The recheck after allocation + /// closes both the failed-check/registration and the fence-allocation/release races. A + /// waiter that goes away leaves its entry for the next notification to signal. + pub(crate) fn wait(&self, observed: u64) -> Result> { + if self.observe() != observed { + return Ok(None); + } + let fence = fence::independent(&self.fences)?; + let mut entry = KBox::new( + Entry { fence: fence.clone(), next: None }, + GFP_KERNEL, + )?; + let mut state = self.state.lock(); + if state.generation != observed { + return Ok(None); + } + entry.next = state.waiters.take(); + state.waiters = Some(entry); + Ok(Some(Fence::from_fence(&fence))) + } + + /// Blocks until a notification after `observed`, for at most `timeout_ms`. For callers + /// outside the scheduler that own no other wait. + pub(crate) fn block(&self, observed: u64, timeout_ms: u32, interruptible: bool) -> Result { + let Some(fence) = self.wait(observed)? else { return Ok(()); }; + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + interruptible, + kernel::time::msecs_to_jiffies(timeout_ms) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + Ok(()) + } + + /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence + /// callbacks run outside the domain lock. + pub(crate) fn notify(&self) { + let mut waiters = { + let mut state = self.state.lock(); + state.generation = state.generation.wrapping_add(1); + state.waiters.take() + }; + while let Some(mut entry) = waiters { + waiters = entry.next.take(); + entry.fence.signal(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea02..8f3a03fbb8dba3 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } @@ -1119,3 +1131,51 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } + +impl BufferIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", + self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, + self.entries.iter().filter(|entry| entry.owner).count(), + self.entries.iter().filter(|entry| entry.references != 0).count()) + } +} +impl MetricsIds { + pub(crate) fn used(&self) -> u32 { + self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() + - (128 - fw::METRICS_SLOTS as u32) + } +} + +#[kunit_tests(asahi_g17_buffer_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_build_and_fault_retirement_return_parameter_buffers() { + let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); + for _ in 0..BUFFER_IDS { + clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); + } + assert_eq!(ids.allocate_render(), Err(ENOSPC)); + for (index, id) in clients.into_iter().enumerate() { + if index % 2 == 0 { + // Failed constructor never installed the graph owner. + ids.release(id).unwrap(); + } else { + ids.reserve_owner(id).unwrap(); + ids.release(id).unwrap(); + ids.acquire_owner(id).unwrap(); + // Close cannot return a still-owned failed command. + ids.unreserve_owner(id).unwrap(); + assert!(!ids.entries[id as usize].free()); + ids.release(id).unwrap(); + } + } + assert!(ids.entries.iter().all(|entry| entry.free())); + assert_eq!(ids.returned_count, BUFFER_IDS); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c935..64c88da8eca149 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at +/// the default parameter-buffer sizes. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs new file mode 100644 index 00000000000000..57406e5487a372 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Name capacity refusals at their source, before a common submit or fence +//! failure handler loses their provenance. Counters are device-lifetime +//! independent diagnostics, never admission or ownership state. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Site { + ComputeReserve, + PhysicalQids, + RenderSlots, + LogicalQueues, + ComputeSlots, + QosIds, + ComputeItems, + RetiredRenderBindings, + RenderPoolIds, + RenderWorkSlots, + ParameterBuffers, + MetricsIds, + LowerVa, + UpperVa, + AliasVa, + ExecutionContexts, +} + +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; +static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; + +/// First and power-of-two refusals per site keep a pressure storm bounded in +/// the log. This does not change whether the caller retries or rejects work. +#[track_caller] +pub(super) fn refused(site: Site) -> Error { + let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); + if count.is_power_of_two() { + let at = core::panic::Location::caller(); + pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); + } + ENOSPC +} + +/// Read-only diagnostics hold the live device session while sampling its pools. +/// They never wake firmware, reclaim a resource, or change admission state. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), + u8::from(self.0.crashed.load(Ordering::Acquire)))?; + let uat = { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; + firmware.queues.write_capacity(f)?; + firmware.pool.write_capacity(f)?; + writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; + firmware.uat.clone() + }; + // Mapping destruction can issue firmware flushes while holding VM + // locks. Never take a VM/TTBAT lock with the device mutex held here. + // The retained UAT owns valid allocator metadata even during shutdown. + writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let mut upper = uat.geometry().kernel_range(); + upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + for (name, vm, range) in [ + ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), + ("dynamic_upper_bytes", uat.kernel_vm(), upper), + ] { + match vm.capacity(range.clone()) { + Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, + Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, + } + } + for (site, count) in [ + Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, + Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, + Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, + ].iter().zip(REFUSALS.iter()) { + writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a1661..d96924cbbf9f91 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -138,6 +138,7 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 3dfa760969224e..c65f7de6ac9157 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,6 +12,8 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; +const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); + const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -121,7 +123,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -131,7 +133,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if header.cmd_type as u32 + let payload = if u32::from(header.cmd_type) == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -238,7 +240,7 @@ impl UscWindow { } require_range( space, - self.base + (tagged_offset & !7) as u64, + self.base + u64::from(tagged_offset & !7), 4, Access::Read, ) @@ -306,12 +308,14 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { + long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -323,6 +327,19 @@ pub(crate) enum Validated { }, } +impl Validated { + pub(crate) fn long_job(&self) -> bool { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, + } + } + pub(crate) fn allow_long_job(&mut self) { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, + } + } +} + impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -334,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -407,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -440,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -447,6 +470,8 @@ impl Payload { } } Ok(Validated::Render { + long_job: super::timeout::select(false, + ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -454,8 +479,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 + let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; + if cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -470,14 +495,19 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { + long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -489,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231dba..02e7bbc5c33dc8 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -27,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +61,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,20 +83,13 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) } + pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } + fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } @@ -144,7 +138,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +161,84 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + +/// Republications of one command whose earlier publications firmware dropped unexecuted. +pub(crate) const MAX_REPLAYS: u8 = 3; + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,8 +246,9 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -185,11 +259,21 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, + /// A recovery blamed this command. Unlike `class`, which a later + /// classification can change, this never clears. + blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { + #[expect( + clippy::too_many_arguments, + reason = "Completion construction takes distinct queue, context, and lifetime owners." + )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -200,6 +284,7 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute @@ -216,7 +301,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -227,6 +312,7 @@ impl Completion { terminal, compute, feed, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -235,6 +321,8 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), + blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -270,6 +358,38 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { self.note_input_error(error); } + } + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { sources.check()?; } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } @@ -278,14 +398,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -305,11 +425,14 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < 3).then_some(count + 1) + (count < MAX_REPLAYS).then_some(count + 1) }) .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -335,7 +458,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -384,6 +507,49 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } + pub(crate) fn guilty(&self) -> bool { + self.class.load(Ordering::Acquire) == 2 + } + + /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, + /// so it is never published again. + pub(crate) fn note_blamed(&self) { + self.blamed.store(true, Ordering::Release); + } + + pub(crate) fn blamed(&self) -> bool { + self.blamed.load(Ordering::Acquire) + } + + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -401,6 +567,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -408,16 +580,24 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(); + lease.quarantine(self.spared()); } } /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -427,6 +607,7 @@ impl Completion { } fn release_healthy(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -434,19 +615,28 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { result = Err(error); } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { - if self.spared() { - self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); - } + // Spared work cancelled after another VM's recovery is not this client's device loss. + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -480,8 +670,8 @@ impl Completion { } if self.spared() { self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44beef..0d3948a2b69ffb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; Ok(QosLease { @@ -114,13 +114,22 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, work: Arc>, work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -133,6 +142,7 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -187,19 +197,42 @@ impl Context { scheduler, scheduler_generation, published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), + state_returns, work, work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -329,6 +362,7 @@ pub(crate) struct WorkStateLease { word: AtomicU32, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { Self { context, @@ -348,8 +382,9 @@ impl WorkStateLease { Err(EBUSY) }; }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + let initialized = self.context.work.lock().initialize(candidate); + if let Err(error) = initialized { + self.context.release_state(candidate); return Err(error); } match self @@ -358,11 +393,15 @@ impl WorkStateLease { { Ok(_) => Ok(candidate), Err(installed) => { - self.context.states.release(candidate); + self.context.release_state(candidate); Ok(installed) } } } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns + } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } @@ -387,7 +426,8 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context.work.lock().render.get(pair).ok_or_else(|| + super::capacity::refused(super::capacity::Site::RenderWorkSlots)) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -402,7 +442,51 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word); + } + } +} + +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32) { + if self.states.release(word) { + self.state_returns.notify(); + } + } +} + +impl QosIds { + pub(crate) fn used(&self) -> u32 { + (*self.used.lock() & !3u128).count_ones() + } +} + +#[kunit_tests(asahi_g17_qos_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_client_construction_returns_every_qos_id() { + let ids = QosIds::new().unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); + for _ in 0..126 { + clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 126); + assert!(matches!(ids.acquire(), Err(ENOSPC))); + drop(clients); + assert_eq!(ids.used(), 0); + assert_eq!(*ids.used.lock(), 3); } } } + +impl Context { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { + writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", + scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 6c48367e2d6864..595c7154d42edc 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -69,3 +69,12 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } + +impl WorkStates { + pub(super) fn used(&self) -> usize { + self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() + } +} +impl RenderSlots { + pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } +} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs new file mode 100644 index 00000000000000..02bcc66de8f51c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/data_segment.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2026 The Gravity Linux Contributors +// +// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and +// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. + +//! Crash log buffers in the GPU firmware's data segment. +//! +//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it +//! by physical address inside its data segment, the `fw-data` reserved region, which the boot +//! loader keeps for the loaded firmware until reboot. + +use kernel::{ + c_str, device, + io::{ + self, + IoSysMap, + Region, // + }, + prelude::*, +}; + +/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. +pub(super) struct DataSegmentLog { + mem: io::mem::Mem, + offset: usize, + size: usize, +} + +impl DataSegmentLog { + /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which + /// must lie inside the `fw-data` region of `dev`. + pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { + return Err(EINVAL); + } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + Ok(Self { mem, offset, size }) + } + + /// The CPU view of the buffer. + pub(super) fn buf(&mut self) -> Result> { + // SAFETY: map() checked that the range lies inside the mapping. + let base = unsafe { self.mem.ptr().add(self.offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5abd494eeb1f58..0bc1d5024566e7 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -17,6 +17,14 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// A replay's original host inputs must still be signalled. Treat an unmet +/// input as a failed replay, never as a wait under the device mutex. +pub(crate) fn replay_status(status: i32) -> Result { + if status == 0 { Err(EBUSY) } + else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else { Ok(()) } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c813..9508a827be4b1b 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 4cd05fde165812..760f352046192b 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -147,6 +147,10 @@ impl Requests { } } + /// Add a policy to the pinned requests. + /// + /// # Safety + /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -179,7 +183,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } } @@ -193,7 +197,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + bindings::FREQ_QOS_MIN_DEFAULT_VALUE }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -211,7 +215,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } applied } @@ -235,7 +239,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } fn clear(mut self: Pin<&mut Self>) { @@ -364,6 +368,13 @@ impl Feed { } } + /// Process a cpufreq policy notification. + /// + /// # Safety + /// + /// `block` must be this pinned Feed's registered notifier, and `data` must + /// point to the live policy supplied by the cpufreq notifier core. + /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -387,19 +398,19 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as _; + return bindings::NOTIFY_DONE as i32; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as _ + bindings::NOTIFY_OK as i32 } fn now() -> u64 { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 3414e36db9d365..8245e8bca71e65 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,10 +7,10 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, - dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, new_mutex, prelude::*, str::CStr, @@ -19,6 +19,16 @@ use kernel::{ pub(crate) struct CompletionFence; +unsafe extern "C" { + fn asahi_g17_signal_fence(fence: *mut bindings::dma_fence); +} + +fn signal_exported(fence: &dyn RawDmaFence) { + // SAFETY: The borrowed fence remains owned throughout synchronous signaling. + // Ready completion scopes are drained iteratively after its lock is released. + unsafe { asahi_g17_signal_fence(fence.raw()) }; +} + #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -48,106 +58,274 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { Ok(Fence::from_fence(&fence)) } -/// All command members of an ioctl share its work-state node and aggregate fence. +/// The existing aggregate-fence allocation owns the existing submission state. +/// Submission never owns this fence, so exported references cannot form a cycle. +pub(crate) struct ExportFence { + source: Option>, + scoped: bool, +} +static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); + +#[vtable] +impl FenceOps for ExportFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-aggregate") + } +} + +#[inline(never)] +fn export_fallback(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; + // SAFETY: Unique, initialized and still private. Only this constructor + // publishes ExportFence objects; its payload is immutable after sharing. + let ops = unsafe { + (*fence.raw()).context = bindings::dma_fence_context_alloc(1); + (*fence.raw()).ops as usize + }; + let previous = EXPORT_OPS.load(Ordering::Acquire); + if previous == 0 { + match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => {}, + Err(previous) if previous == ops => {}, + Err(_) => return Err(EIO), + } + } else if previous != ops { + return Err(EIO); + } + Ok(fence) +} + +/// Caller retains the leaf fence throughout lookup. Exact registered operations +/// identify the repr(C) FenceObject payload, not a driver or timeline name. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) + -> Option> +{ + let expected = EXPORT_OPS.load(Ordering::Acquire); + if expected == 0 { return None; } + // SAFETY: The caller owns a live fence reference. ExportFence has a unique + // timeline callback and nontrivial payload drop. No cast precedes the exact + // registered vtable check. FenceObject begins with its dma_fence. + let object = unsafe { + if (*raw).ops as usize != expected { return None; } + &*(raw as *const FenceObject) + }; + object.source.as_ref().filter(|source| source.firmware_source).cloned() +} + +struct State { + work: Option>, + ready: Option, +} + +// One Member per parsed command, plus the enqueue guard. Exported references +// retain the allocation but do not contribute to either of these counters. +kernel::static_assert!(crate::file::MAX_COMMANDS_PER_SUBMISSION < u8::MAX as u32); + /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] pub(crate) struct Submission { - fence: UserFence, - /// Successful retirement drops only this bookkeeping owner; runtime owners - /// and failed submissions keep their work-state allocation. #[pin] - work_state: Mutex>>, - remaining: AtomicU32, + state: Mutex, + remaining: AtomicU8, error: AtomicI32, + /// Live Member objects, plus the enqueue guard, independently of signalling. + owners: AtomicU8, + firmware_source: bool, + /// Set under the device mutex when the only member's successful retirement + /// is queued, before its fence is signalled off-lock: a consumer published + /// in between needs no wait on it. + retired: AtomicBool, } impl Submission { - fn new(fence: UserFence, context: Arc) -> Result> { - let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - fence, - work_state <- new_mutex!(Some(work_state), "G17 submission work state"), - remaining: AtomicU32::new(1), - error: AtomicI32::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) - } - - /// Add one command before enqueue ends. The parser bounds the count to 64. - pub(crate) fn member(self: &Arc) -> Member { - self.remaining.fetch_add(1, Ordering::Relaxed); - Member { - submission: self.clone(), - complete: AtomicBool::new(false), - } + fn new(context: Arc, firmware_source: bool) -> Result> { + let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Self::with_work(Some(work), firmware_source) } - - fn member_done(&self, result: Result) { - if let Err(error) = result { - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } + fn with_work(work: Option>, firmware_source: bool) -> Result> { + Arc::pin_init(pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + retired: AtomicBool::new(false), + }), GFP_KERNEL) + } + pub(super) fn source_status(&self) -> Result { + let complete = self.remaining.load(Ordering::Acquire) == 0 + || self.retired.load(Ordering::Acquire); + let error = self.error.load(Ordering::Acquire); + if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + Ok(complete) + } + pub(super) fn same_vm(&self, status: &Arc) -> bool { + self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } + fn set_ready(&self, ready: Fence) { + if !self.firmware_source { return; } + let mut state = self.state.lock(); + if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + } + fn note_error(&self, error: Error) -> bool { + self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + } + fn release_state(&self) { + let old = { + let mut state = self.state.lock(); + (state.work.take(), state.ready.take()) + }; + drop(old); + } + fn owner_done(&self) { + // Every hardware owner retains its Completion/Member through exact + // retirement or stop. Exported fence FDs must not retain failed pins. + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + } + fn member_done(&self, fence: &UserFence, result: Result) { + if let Err(error) = result { self.note_error(error); } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { - self.fence.set_error(Error::from_errno(error)); - } else { - // Scheduler cleanup can wait behind submission on the same worker. - // Once every member and enqueue have succeeded, retained packets - // must not prevent new submissions from acquiring work-state nodes. - let retired = self.work_state.lock().take(); - drop(retired); - } - self.fence.signal(); + if error != 0 { fence.set_error(Error::from_errno(error)); } + else { self.release_state(); } + if fence.scoped { signal_exported(fence); } else { fence.signal(); } } } +} +#[derive(Clone)] +pub(crate) struct Aggregate(UserFence); +impl Aggregate { + fn source(&self) -> &Arc { + // Only Outputs::publish constructs Aggregate, after setting the payload. + self.0.source.as_ref().expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { Fence::from_fence(&self.0) } + pub(crate) fn member(&self) -> Member { + self.source().remaining.fetch_add(1, Ordering::Relaxed); + self.source().owners.fetch_add(1, Ordering::Relaxed); + Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + } fn finish_enqueue(&self, result: Result) { - // A dropped, never-enqueued member reports cancellation. The ioctl's actual - // failure takes precedence, while published members still retain their work. - if let Err(error) = result { - self.error.store(error.to_errno(), Ordering::Release); - } - self.member_done(result); + if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + self.source().member_done(&self.0, result); + self.source().owner_done(); } } -/// One command's contribution to its submission, including construction failure. pub(crate) struct Member { - submission: Arc, + aggregate: Aggregate, complete: AtomicBool, } - impl Member { + pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } pub(crate) fn work_state(&self) -> Result> { - self.submission - .work_state - .lock() - .as_ref() - .cloned() - .ok_or(ECANCELED) + self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) + } + pub(super) fn is_source(&self, source: &Arc) -> bool { + self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } + pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn note_retired(&self) { + let source = self.aggregate.source(); + if source.firmware_source { source.retired.store(true, Ordering::Release); } } - pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.submission.member_done(result); + self.aggregate.source().member_done(&self.aggregate.0, result); } } } +#[kunit_tests(asahi_g17_aggregate_promise)] +mod tests { + use super::*; + use kernel::sync::LockClassKey; + static KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + + fn contexts() -> FenceContexts { + FenceContexts::new(1, c_str!("asahi_promise_test"), KEY).unwrap() + } + fn aggregate(contexts: &FenceContexts) -> Aggregate { + let mut fence = export_fallback(contexts).unwrap(); + fence.source = Some(Submission::with_work(None, true).unwrap()); + Aggregate(fence.into()) + } + + #[test] + fn exact_export_identity_recognizes_unpublished_source() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let plain = independent(&contexts).unwrap(); + // SAFETY: Both references remain owned throughout the checked lookup. + assert!(unsafe { exported_source_raw(plain.raw()) }.is_none()); + let source = unsafe { exported_source_raw(aggregate.0.raw()) }.unwrap(); + assert!(Arc::ptr_eq(&source, aggregate.source())); + assert_eq!(source.source_status().unwrap(), false); + aggregate.finish_enqueue(Ok(())); + assert_eq!(source.source_status().unwrap(), true); + } + + #[test] + fn failed_export_releases_state_after_last_runtime_owner() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let source = aggregate.source().clone(); + source.state.lock().ready = Some(Fence::from_fence(&independent(&contexts).unwrap())); + let member = aggregate.member(); + aggregate.finish_enqueue(Ok(())); + member.complete(Err(EIO)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert!(source.state.lock().ready.is_some()); + drop(member); + // Keep both the exported fence and source state alive, as userspace can. + assert!(source.state.lock().ready.is_none()); + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } + + #[test] + fn actual_enqueue_error_wins_over_dropped_member_cancellation() { + let aggregate = aggregate(&contexts()); + drop(aggregate.member()); + aggregate.finish_enqueue(Err(ENOMEM)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + } + + #[test] + fn rejected_private_fence_never_exports_source_state() { + let fence = export_fallback(&contexts()).unwrap(); + fence.set_error(ENOMEM); + fence.signal(); + // SAFETY: The unique fence owns its initialized allocation. + assert!(unsafe { exported_source_raw(fence.raw()) }.is_none()); + } + + #[test] + fn maximum_submission_keeps_all_members_and_enqueue_guard() { + let aggregate = aggregate(&contexts()); + let source = aggregate.source().clone(); + let mut members: [Option; crate::file::MAX_COMMANDS_PER_SUBMISSION as usize] = + core::array::from_fn(|_| Some(aggregate.member())); + assert_eq!(source.remaining.load(Ordering::Acquire), 65); + assert_eq!(source.owners.load(Ordering::Acquire), 65); + aggregate.finish_enqueue(Ok(())); + for member in &members { + assert_eq!(super::super::job::fence_status(&aggregate.fence()), 0); + member.as_ref().unwrap().complete(Ok(())); + } + assert!(super::super::job::fence_status(&aggregate.fence()) > 0); + assert_eq!(source.remaining.load(Ordering::Acquire), 0); + for member in &mut members { drop(member.take()); } + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } +} impl Drop for Member { fn drop(&mut self) { self.complete(Err(ECANCELED)); + self.aggregate.source().owner_done(); } } @@ -155,83 +333,73 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(fence)); - } + } else { sync.syncobj.replace_fence(Some(fence)); } } } -/// Keeps resolved output syncobjs reachable on every fallible submission path. -/// Input syncobjs stay in the same vector, avoiding a separate output allocation. pub(crate) struct Outputs { - fallback: UserFence, + fallback: Option>, syncs: KVec, input_count: usize, - submission: Option>, + submission: Option, result: Result, + cumulative: Option, } - impl Outputs { - pub(crate) fn new( - contexts: &FenceContexts, - mut syncs: KVec, - input_count: usize, - allocation_failure: &Fence, - ) -> Result { - if input_count > syncs.len() { - return Err(EINVAL); - } - let fallback = match independent(contexts) { + pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, + input_count: usize, allocation_failure: &Fence) -> Result + { + if input_count > syncs.len() { return Err(EINVAL); } + let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); + let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - // Syncobjs and timeline-chain storage are already resolved. - // No work has been accepted; publish the pre-signaled ENOMEM - // fence without another allocation before returning the error. - install(syncs.drain(input_count..), allocation_failure); + if let Some(bundle) = &mut cumulative { + bundle.fail(allocation_failure); + } else { + install(syncs.drain(input_count..), allocation_failure); + } return Err(error); } }; - Ok(Self { - fallback, - syncs, - input_count, - submission: None, - result: Err(ECANCELED), - }) - } - - pub(crate) fn inputs(&self) -> &[SyncItem] { - &self.syncs[..self.input_count] + Ok(Self { fallback: Some(fallback), syncs, input_count, + submission: None, result: Err(ECANCELED), cumulative }) } - - /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, context: Arc) -> Result> { - let submission = Submission::new(self.fallback.clone(), context)?; - install(self.syncs.drain(self.input_count..), &submission.fence()); + pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } + pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + let source = Submission::new(context, firmware_source)?; + let mut fence = self.fallback.take().ok_or(EINVAL)?; + fence.source = Some(source); + fence.scoped = self.cumulative.is_some(); + let submission = Aggregate(fence.into()); self.submission = Some(submission.clone()); + if let Some(bundle) = &mut self.cumulative { + bundle.publish(&submission.fence())?; + self.syncs.truncate(self.input_count); + } else { + install(self.syncs.drain(self.input_count..), &submission.fence()); + } Ok(submission) } - - /// The caller first records terminal VM errors and settles prior failed work. - pub(crate) fn finish(mut self, result: Result) -> Result { - self.result = result; - result - } + pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } } - impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&submission.fence()); + } submission.finish_enqueue(self.result); return; } - if let Err(error) = self.result { - self.fallback.set_error(error); + if let Some(fallback) = self.fallback.take() { + if let Err(error) = self.result { fallback.set_error(error); } + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&Fence::from_fence(&fallback)); + } else { + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + } + fallback.signal(); } - install( - self.syncs.drain(self.input_count..), - &Fence::from_fence(&self.fallback), - ); - self.fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc0..9015dc82e40765 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) } } impl Drop for RenderId { @@ -69,19 +69,84 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -100,7 +165,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - fn page_list_va(&self) -> u64 { + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -144,14 +209,17 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, @@ -522,6 +590,18 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } + self.write_pages(&pages)?; + self.pages = Some(pages); + Ok(()) + } + /// Preserve both sets of pages on failure. The caller owns the consumed + /// release and returns old pages for destruction outside the device mutex. + pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { + self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; + core::mem::swap(&mut self.pages, pages); + Ok(()) + } + fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -540,9 +620,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); - self.pages = Some(pages); Ok(()) } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -590,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -627,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -692,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) @@ -742,3 +853,30 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } + +impl RenderIds { + pub(crate) fn used(&self) -> u32 { + self.used[0].load(Ordering::Acquire).count_ones() + + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() + } +} + +#[kunit_tests(asahi_g17_render_id_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn rejected_pool_construction_returns_all_render_ids() { + let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); + for _ in 0..127 { + pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 127); + assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); + drop(pools); + assert_eq!(ids.used(), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index ec6797637bdbe1..9d430d50db365e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c4cf6960db296d..524132c4ae7beb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -498,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: first as u8, + first: u8::from(first), }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0633da7f435411..b1afdad816b6b7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u16::MAX as u64; - let cliques = (layout >> 24) & u8::MAX as u64; - let user = (layout >> 32) & u16::MAX as u64; - let driver = (layout >> 48) & u16::MAX as u64; + let depth = layout & u64::from(u16::MAX); + let cliques = (layout >> 24) & u64::from(u8::MAX); + let user = (layout >> 32) & u64::from(u16::MAX); + let driver = (layout >> 48) & u64::from(u16::MAX); if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,13 +152,42 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster of six cores (A18 Pro). + pub(crate) const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + pub(crate) const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +312,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +336,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -347,14 +379,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +396,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } @@ -474,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -489,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 1f25bd0fc1c0dc..3092e599caa846 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -591,31 +608,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va - .checked_add(self.size as u64)? + let start = va & !mask; + let span = va + .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +657,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +678,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -676,14 +704,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +761,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +791,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) @@ -1169,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1197,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1213,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1224,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1234,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1285,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1319,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1364,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1411,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 00000000000000..c237896a49fb1a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index e07fbfb485a10a..ca2b6604a302b5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. @@ -297,7 +311,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, }) } @@ -332,7 +346,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | ((self.count - 1) as u64) << Self::COUNT_SHIFT + | u64::from(self.count - 1) << Self::COUNT_SHIFT } } @@ -444,8 +458,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | (args.qid as u64) << Self::QID_SHIFT - | (args.priority as u64) << Self::PRIORITY_SHIFT; + | u64::from(args.qid) << Self::QID_SHIFT + | u64::from(args.priority) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -478,9 +492,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: mask as u64 - | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT - | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + dependency_control: u64::from(mask) + | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT + | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index b6bb6aaea860ff..1cb46fce2d6028 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -30,7 +33,7 @@ pub(crate) mod qos; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; + unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 48b110a4193e3b..5708a2251a062d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -94,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -115,26 +121,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +152,62 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + /// Offsets of the two host words a [`PointerLayout`] may name. + const WORD_40: usize = core::mem::offset_of!(Self, word_40); + const WORD_50: usize = core::mem::offset_of!(Self, word_50); + + /// Whether `offset` is one of the host words a [`PointerLayout`] may name. + const fn is_host_word(offset: usize) -> bool { + offset == Self::WORD_40 || offset == Self::WORD_50 + } + + /// Whether `layout` names host words of the block, and two different ones when it has a + /// recovery target. Every firmware variant's layout is checked at build time. + pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { + match layout.recovery_target { + None => Self::is_host_word(layout.producer), + Some(target) => { + Self::is_host_word(layout.producer) + && Self::is_host_word(target) + && target != layout.producer + } + } + } /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Result { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target)?.store(u32::MAX, Ordering::Relaxed); + } + Ok(block) + } + + /// The host word at `offset`, which must be one a [`PointerLayout`] may name. + fn word(&self, offset: usize) -> Result<&AtomicU32> { + match offset { + Self::WORD_40 => Ok(&self.word_40), + Self::WORD_50 => Ok(&self.word_50), + _ => Err(EINVAL), } } @@ -174,12 +216,14 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { + let producer = self.word(layout.producer)?; self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + producer.store(base, Ordering::Relaxed); + Ok(()) } } @@ -271,12 +315,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [args.policy.priority() as u32; 2], + priority: [u32::from(args.policy.priority()); 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: args.policy.policy() as u32, + policy: u32::from(args.policy.policy()), unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -307,12 +351,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [policy.priority() as u32; 2]; + let priority = [u32::from(policy.priority()); 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != policy.policy() as u32 { - self.policy = policy.policy() as u32; + if { self.policy } != u32::from(policy.policy()) { + self.policy = u32::from(policy.policy()); } } } @@ -480,7 +524,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - binding.fragment_qid as u32, + u32::from(binding.fragment_qid), ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -491,14 +535,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: args.qid as u16, - install: args.install as u16, + qid: u16::from(args.qid), + install: u16::from(args.install), unk_1c: 0, - priority: args.policy.priority() as u16, + priority: u16::from(args.policy.priority()), unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed_valid: u32::from(args.completion_seed.is_some()), completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -506,13 +550,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: args.qos_update as u32, - qos_slot: args.qos_slot as u16, + qos_update: u32::from(args.qos_update), + qos_slot: u16::from(args.qos_slot), unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: args.context_update as u16, + context_update: u16::from(args.context_update), unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -525,6 +569,94 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp, in addition to the kick-entry dependency. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u64, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id, + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -564,7 +696,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: qid as u16, + qid: u16::from(qid), count: 1, priority, timestamp: timestamp.get(), @@ -603,7 +735,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: qid as u16, + qid: u16::from(qid), previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 85a2385b421b81..aa8693bf7140b7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,53 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// One cluster (A18 Pro). + pub(crate) const ONE_CLUSTER: Deflake = Deflake { + sub: [0x20, 0x2a0], + viewport: true, + }; + /// Two clusters (M4 Pro). + pub(crate) const TWO_CLUSTERS: Deflake = Deflake { + sub: [0xa0, 0x820], + viewport: false, + }; +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +147,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +309,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +364,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -357,12 +411,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = pass.width as u64; - let height = pass.height as u64; + let width = u64::from(pass.width); + let height = u64::from(pass.height); let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / pass.utile_width as u64; - let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles_x = TILE_SIZE / u64::from(pass.utile_width); + let utiles_y = TILE_SIZE / u64::from(pass.utile_height); let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -411,7 +465,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -441,19 +495,19 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u32::MAX as u64, + false => spec & u64::from(u32::MAX), }; - let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); Ok(Self { - utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | pass.samples_log2 as u64, + utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | u64::from(pass.samples_log2), tile_config, layer_mode, - tib_blocks: (pass.sample_size as u64 - * pass.utile_width as u64 - * pass.utile_height as u64 + tib_blocks: (u64::from(pass.sample_size) + * u64::from(pass.utile_width) + * u64::from(pass.utile_height) * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, @@ -491,9 +545,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let deflake = user_offset32(r.deflake_va)? as u64; - let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; - let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let [deflake_20, deflake_2a0] = args.deflake.sub; + let deflake = u64::from(user_offset32(r.deflake_va)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -561,7 +616,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -573,7 +628,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -581,6 +636,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +766,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -609,8 +792,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, p.merge_upper_x as u64), - RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), + RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -625,15 +808,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, p.depth_clear as u64), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15301, u64::from(p.depth_clear)), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -682,11 +865,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -716,7 +899,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -742,13 +925,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -760,13 +943,13 @@ fn partial_load_registers( RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -902,7 +1085,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,12 +1103,14 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -945,7 +1131,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -969,7 +1158,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = fragment_busy as u32; + self.fragment_busy = u32::from(fragment_busy); self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -980,11 +1169,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1363,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,13 +1446,15 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; @@ -1289,12 +1488,15 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = (args.ordinal == 0) as u32; + desc.first_kick = u32::from(args.ordinal == 0); desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 00000000000000..2f7b5987333688 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, +} + +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, + init_ack: InitAck::Message, + power_on_done: true, + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, + lead_commands: false, + programs: super::render::RegisterPrograms::G17P, +}; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; + +const _: () = { + assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); +}; diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 20daaab85b8325..4d9e99d9a9cc0c 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -3,11 +3,21 @@ //! Attachment overlap ordering across physical render pairs and shared images. //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. +//! +//! Within one VM, a render of another logical owner is ordered at publication: +//! the device mutex totally orders publications, and the later render's +//! fragment kick waits in firmware for the newest overlapping fragment kick of +//! every other pair. Admission never waits for such an overlap. Only a +//! cross-VM overlap through a shared object waits on the host for retirement, +//! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; +/// Firmware barriers one render publication may add for same-VM overlaps, +/// one per other pair. +pub(crate) const PUBLICATION_BARRIERS: usize = 7; #[derive(Clone, Copy)] struct Span { @@ -56,7 +66,12 @@ impl Footprint { self.shared |= self.count != before; } - fn overlaps(&self, other: &Self) -> bool { + /// Whether a publication orders an overlap with `other` in firmware. + pub(crate) fn same_vm(&self, other: &Self) -> bool { + self.vm == other.vm + } + + pub(crate) fn overlaps(&self, other: &Self) -> bool { if self.vm != other.vm && !(self.shared && other.shared) { return false; } @@ -86,6 +101,13 @@ pub(crate) struct Live<'a> { /// A queue prefix already orders covered predecessors. Pending arrivals order /// different owners fairly without imposing order on unrelated attachments. +/// +/// A same-VM overlap with another owner's admitted render needs no wait; its +/// publication carries one barrier per other pair. That count stays within +/// [`PUBLICATION_BARRIERS`] because such a render is admitted only while fewer +/// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: +/// of any set of renders overlapping one publication, the last admitted saw all +/// the others live. pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -94,10 +116,29 @@ pub(crate) fn plan<'a>( live: impl IntoIterator>, ) -> Plan { let mut selected = None; + let mut ordered = false; + let mut pairs = [0u8; PUBLICATION_BARRIERS]; + let mut count = 0; + let mut crowded = false; for entry in live { + let foreign = entry.owner != owner && footprint.same_vm(entry.footprint); + if let Some(slot) = entry.slot.filter(|_| foreign) { + if !pairs[..count].contains(&slot) { + if count == PUBLICATION_BARRIERS { + crowded = true; + } else { + pairs[count] = slot; + count += 1; + } + } + } if entry.quarantined || !footprint.overlaps(entry.footprint) { continue; } + if foreign { + ordered |= entry.slot.is_some(); + continue; + } let Some(slot) = entry.slot else { if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { return Plan::Wait; @@ -116,5 +157,8 @@ pub(crate) fn plan<'a>( Some(_) => return Plan::Wait, } } + if ordered && crowded { + return Plan::Wait; + } selected.map_or(Plan::Any, Plan::Pair) } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09eb..7e0c36ece43ad5 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -36,6 +38,26 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { slots } else { 0x800 } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -121,20 +143,12 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - self.record as *const _ as *const u8, + core::ptr::from_ref(self.record).cast::(), self.destination, core::mem::size_of_val(self.record), ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -155,6 +169,15 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -174,6 +197,8 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, + /// GPU clusters, as the GPU reports them. + clusters: u32, } impl BuildEnv<'_> { @@ -235,6 +260,11 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +277,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -363,7 +397,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); } Ok(()) })?; @@ -372,6 +406,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -488,7 +528,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -502,7 +542,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -522,7 +562,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -541,16 +581,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -601,7 +631,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -649,21 +679,118 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. + pub(super) fn port_views( + &self, + mailbox: *const super::Mailbox, + ) -> Result { + use super::fw::{channels, queue::{DataMaster, Policy}}; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = |offset: usize| -> Result<*const AtomicU32> { + Ok(state.word(base + offset)?) + }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize + * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; + Ok(super::port::Views { + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( &self, + start: usize, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( + &self, + seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + let mut newest = None; + for slot in 0..COMPUTE_COMPLETION_SLOTS { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })?; + }; + let control = completion.read(0)?; + seen[slot] = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } + } + record(completion)?; } - Ok(()) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -694,7 +821,12 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + clusters: u32, + active_cores: u32, + variant: &super::fw::Variant, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -703,9 +835,13 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + clusters, }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; + if variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } + self.map_register_windows(&env, cfg.windows)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -713,31 +849,41 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -939,12 +1085,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for slot in 0..2 { + for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - completion_rings[slot] = abi::CompletionRing { + *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -988,13 +1134,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1002,7 +1148,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1050,8 +1196,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1107,7 +1253,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1150,7 +1296,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1180,37 +1326,65 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + /// The GPU's operating points: `count` of them, each with one voltage cell for the whole + /// GPU or one per cluster. + fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + let cells = opps.first().map_or(0, |opp| opp.cells); + if cells != 1 && u32::try_from(cells) != Ok(clusters) { + dev_err!( + dev.as_ref(), + "GPU operating points have {} voltage cells, expected 1 or {}\n", + cells, + clusters + ); + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1218,7 +1392,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1247,9 +1421,14 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1261,7 +1440,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1336,7 +1515,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1351,8 +1530,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1363,20 +1546,20 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; @@ -1391,21 +1574,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1417,17 +1600,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1435,17 +1622,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 00000000000000..85aa6468fa311f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6dd..647b20e9753219 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,8 +13,11 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; -const RENDER_CREDITS: u32 = 32; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -143,29 +146,48 @@ impl Packet { } } + /// Replay holds the device mutex. Its original inputs were satisfied before + /// first publication; never sleep on a fence that needs event service here. + pub(crate) fn check_replay_dependencies(&self) -> Result { + for fence in &self.inputs { + super::dependency::replay_status(fence_status(fence))?; + } + for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; + Ok(()) + } + /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { + let polls = if self.command.long_job() { 5 } else { 1 }; + let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); + loop { + if self.cancel_requested() || self.completion.status().get() != 0 { + return Err(ECANCELED); + } + if fence_status(fence) != 0 { break; } + if super::now_ns() >= deadline { return Err(ECANCELED); } + // SAFETY: The retained fence owns the pointer throughout the bounded wait. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::time::msecs_to_jiffies(10) as isize, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited > 0 { break; } } check_status(fence)?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -179,6 +201,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -201,7 +230,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |_| 1), + result.map_or_else(|error| error.to_errno(), |()| 1), Ordering::Release, ); } @@ -238,11 +267,15 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } - /// Lazily create or rebind this logical queue's physical compute graph. + /// Validate compute sizing without reserving a physical channel. Queued + /// commands may wait on a host signal submitted after this ioctl returns. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -252,7 +285,9 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); @@ -271,6 +306,8 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, + execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -287,12 +324,15 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, + execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -301,9 +341,21 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } + if super::timeout::renew_execution( + job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, + ) { + return true; + } + // Other work retiring means the GPU is shared, not hung. + if super::timeout::renew_shared( + job.packet.is_published(), super::recovery::since_retired(super::now_ns()), + &mut job.shared_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { - job.victim_renewals += 1; + if deferred { + job.victim_renewals = job.victim_renewals.saturating_add(1); } deferred } @@ -342,6 +394,18 @@ pub(crate) struct Fences { pub(crate) publication: Option>, } +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } +} + impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { @@ -396,7 +460,10 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86de..a1f8f70f597d47 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -17,8 +24,40 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT)] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + +const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); + assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); +}; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -47,19 +86,40 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - compute_publication: u64, + initial_render: [Option; 2], leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - compute_publication: 0, + initial_render: [None; 2], leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -67,6 +127,53 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -83,16 +190,36 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + // The QIDs below the floor are never available to the host. + let total = QID_COUNT - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) + .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { qid: index as u8, owner, @@ -102,16 +229,14 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. - pub(crate) fn next_compute_publication(&mut self) -> Result { - self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.compute_publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -127,6 +252,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -140,6 +272,117 @@ impl QueueIds { } } +#[kunit_tests(asahi_g17_qid_accounting)] +mod accounting_tests { + use super::*; + + fn table() -> KBox { + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() + } + + #[test] + fn render_identities_satisfy_the_compute_reserve() { + let mut ids = table(); + for owner in 1..=FULL_RENDER_RESERVE as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + assert_eq!(ids.leased, QID_COUNT); + } + + #[test] + fn compute_cannot_consume_the_render_minimum() { + let mut ids = table(); + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + ids.cancel_unpublished(tiling).unwrap(); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); + } + + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } + } + + #[test] + fn initial_reservations_yield_but_published_ids_do_not() { + let mut ids = table(); + ids.reserve_initial_render().unwrap(); + for owner in 3..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); + let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); + assert_ne!(first.qid(), second.qid()); + assert!(ids.initial_render.iter().all(Option::is_none)); + assert_eq!(ids.leased, QID_COUNT); + ids.publish(first).unwrap(); + assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT); + ids.cancel_unpublished(second).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT - 1); + } + + #[test] + fn failed_second_render_id_returns_only_the_unpublished_first() { + let mut ids = table(); + for owner in 1..QID_COUNT as u64 { + let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + ids.publish(id).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); + ids.cancel_unpublished(first).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); + assert_eq!(replacement.qid(), first.qid()); + assert_ne!(replacement, first); + assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -147,10 +390,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -186,6 +513,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -220,14 +550,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -255,14 +616,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -272,19 +645,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -297,7 +670,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -353,3 +726,20 @@ impl Queue { }) } } + +impl QueueIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let mut engines = [0usize; 3]; + let mut published = 0; + for entry in self.entries.iter().flatten() { + engines[match entry.id.engine { + DataMaster::Tiling => 0, + DataMaster::Fragment => 1, + DataMaster::Compute => 2, + }] += 1; + published += usize::from(entry.published); + } + writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", + self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9da90d2b0843ab..a0d23c9d69e1e8 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,20 +12,29 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. - +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. + +mod availability; mod buffer; mod channel; mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; +mod data_segment; mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod sync_input; +pub(crate) mod sync_output; +mod trace; mod event; mod feed; mod fence; @@ -35,13 +44,16 @@ mod ids; pub(crate) mod hazard; mod initdata; mod job; +mod capacity; mod kick; mod object; -mod power; +mod port; +pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; +mod replay; mod runtime; pub(crate) mod status; mod teardown; @@ -52,6 +64,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, Ordering, // }; @@ -149,12 +162,63 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, + /// Layouts of the command buffers that follow the GPU's core layout. + pub(crate) core_layout: CoreLayout, +} + +/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. +pub(crate) struct CoreLayout { + /// The cluster and enabled core counts the layouts were measured on, which the GPU must + /// report at probe; `None` where the layouts hold for every configuration of the SoC. + pub(crate) cores: Option<(u32, u32)>, + /// A compute queue's operand-state buffer. + pub(crate) operand_state: fw::compute::OperandState, + /// A render's deflake buffer. + pub(crate) deflake: fw::render::Deflake, +} + +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -197,6 +261,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -207,7 +274,15 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData(data_segment::DataSegmentLog), } impl rtkit::Buffer for CrashLog { @@ -216,7 +291,10 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData(log) => log.buf(), + } } } @@ -225,8 +303,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -305,26 +383,71 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { - return Err(EINVAL); + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); } + // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( + &data.shared.dev, + iova, + size, + )?), }) } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -338,27 +461,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), + }; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -372,7 +500,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -401,24 +529,30 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() - } - - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; - self.state.wait_for_ack() + self.rtkit.with(|rtkit| rtkit.boot()) + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -452,6 +586,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -472,15 +614,25 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + hold: power::Hold, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -490,7 +642,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -500,6 +654,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -519,6 +682,11 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { + // An idle hold keeps the cores powered past registration; it releases them itself. + if self.hold.active() { + self.registration_power_release_pending = false; + return Ok(()); + } power::release(self) } @@ -529,33 +697,84 @@ impl Firmware { } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - match self.effort.idle(now_ns, in_flight) { - power::IdleAction::None => Ok(None), + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { + power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; - Ok(None) + None + } + power::IdleAction::Recheck(delay) => Some(delay), + }; + Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }) + } + + /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A + /// refused control (recovery in progress) is retried on a later check. + fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { + const RETRY_NS: u64 = 1_000_000; + let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); + for _ in 0..2 { + match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + power::HoldAction::None => return None, + power::HoldAction::Recheck(delay) => return Some(delay), + power::HoldAction::Assert => { + if self.set_idle_control(false).is_err() { + return None; + } + self.hold.asserted(); + } + power::HoldAction::Release => { + if self.set_idle_control(true).is_err() { + return Some(RETRY_NS); + } + self.hold.released(); + return None; + } } - power::IdleAction::Recheck(delay) => Ok(Some(delay)), } + None + } + + fn set_idle_control(&mut self, enabled: bool) -> Result { + power::Control::set_idle(self, enabled) } } @@ -610,12 +829,32 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, + /// Layout of renders' deflake buffers for this GPU's cores. + deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, #[pin] @@ -638,6 +877,8 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -727,6 +968,27 @@ impl Shared { drop(retired); } + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changed.notify_all(); + self.renders.notify(); + } + + + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -742,12 +1004,21 @@ impl Shared { } fn drain_reclaims(self: &Arc) { + let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); + reclaimed = true; + } + if reclaimed { + // Context release returns its execution root off-lock. The next + // device pass can now detach idle compute aliases and their QoS + // leases, without depending on a new client's submission/event. + self.queues.notify(); + self.queue_events(); } } @@ -826,6 +1097,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _capacity: Pin>>, } impl Gpu { @@ -836,6 +1108,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -856,21 +1129,46 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; + let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + if cfg + .core_layout + .cores + .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + { + dev_err!( + dev.as_ref(), + "Unsupported GPU core layout: {} clusters with {} cores\n", + id.num_clusters, + active_cores + ); + return Err(ENODEV); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant, + operand_state: &cfg.core_layout.operand_state, + deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, + queues, + renders, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), @@ -880,10 +1178,16 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", capacity::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -892,15 +1196,55 @@ impl Gpu { geometry, ids: Default::default(), shared, + _capacity: capacity, }, GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -932,6 +1276,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -948,11 +1293,20 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -961,12 +1315,24 @@ impl Gpu { secondary, primary, bridge, + port: port::Port::new()?, + seen_submissions: 0, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + hold: power::Hold::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: shared.variant, + queues <- runtime::Registry::with_waiters( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), + ), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -977,45 +1343,97 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, + shared.clusters, + shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; + } + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if shared.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1081,12 +1499,21 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; - params.gpu_generation = hw::GpuGen::G17 as u32; + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; + let variant = self.shared.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1115,9 +1542,18 @@ impl gpu::Gpu for Gpu { true } + /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not + /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 + /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads + /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case + /// failed; all pass with soft faults off). + fn supports_soft_faults(&self) -> bool { + false + } + fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1156,12 +1592,31 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.shared.variant, + )?; let policy = scheduling_policy(priority)?; + let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, + Err(error) if error == ENOSPC => { + // Closed logical queues can still own execution/QoS IDs + // until the scheduler release is consumed and reclaimed. + // Backend construction has rolled back all temporary leases. + let waited = now_ns().saturating_sub(admission_started); + let limit = job::TIMEOUT_MS as u64 * 1_000_000; + if waited >= limit { return Err(EBUSY); } + self.shared.queue_events(); + // A returned queue slot or reclaimed context wakes this. + self.shared + .queues + .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; + } result => break result?, } }; @@ -1185,27 +1640,32 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) + ) -> Result { + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so -/// MEDIUM arrives as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH +/// arrives as 1, MEDIUM as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { + 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae4..f1b22ad5d73310 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,7 +78,17 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + Placement::In(range, align) => { + let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start + && range.end <= crate::hw::t8140::dynamic::LOWER.end { + super::capacity::Site::LowerVa + } else { + super::capacity::Site::UpperVa + }; + this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + if error == ENOSPC { super::capacity::refused(site) } else { error } + })? + } }; this.mapping = Some(mapping); Ok(this) @@ -113,6 +123,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -159,6 +172,20 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) + } + + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -341,6 +368,76 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = (if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + ) + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + ) + }).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) + } else { error })?; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + ).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) + } else { error })?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware @@ -577,3 +674,16 @@ impl Pool { } } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let inner = self.inner.lock(); + for family in 0..POOL_FAMILIES { + let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; + writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, + inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), + entries.iter().flatten().filter(|entry| entry.poisoned).count())?; + } + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + } +} diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs new file mode 100644 index 00000000000000..db043ef9d6f91e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide publication state that physical queues share without the device mutex. +//! +//! A firmware session owns its port, and the views it holds name that session's +//! firmware-shared memory. Code that does not hold the device mutex reaches the port only +//! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every +//! lease before they detach the session. Each piece below is atomic or has its own lock. +//! +//! Lock order: device mutex -> compute queue cell -> the locks of this port. The render +//! publication lock comes first among the port's locks: device mutex -> render publication +//! lock -> admission pool, QoS, mailbox. Its holder never takes the device mutex. + +use super::{ + fw::{ + channels::{WorkSlot, WORK_RING_SLOTS}, + queue::{DataMaster, Policy}, + }, + qos, +}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{ + new_mutex, new_spinlock, + prelude::*, + sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, +}; + +const PRIORITIES: usize = Policy::PRIORITIES as usize; + +/// One priority class's compute work ring: its slot array and the cursors of its two firmware +/// consumers and of the host producer. +#[derive(Clone, Copy)] +pub(super) struct RingView { + pub(super) slots: *mut u8, + pub(super) consumers: [*const AtomicU32; 2], + pub(super) producer: *const AtomicU32, +} + +impl RingView { + fn cursors(&self) -> Result<[u32; 3]> { + // SAFETY: The views name 32-bit words of the session's retained firmware objects, which + // the port's owner keeps alive. Other agents only use single 32-bit accesses. + let values = unsafe { + [ + (*self.consumers[0]).load(Ordering::Relaxed), + (*self.consumers[1]).load(Ordering::Relaxed), + (*self.producer).load(Ordering::Relaxed), + ] + }; + if values.iter().any(|&value| value >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(values) + } + + /// Slots the host may still fill before reaching either consumer. + fn free(&self) -> Result { + let [first, second, producer] = self.cursors()?; + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + Ok(room(first).min(room(second))) + } +} + +/// Raw views of the session's firmware-shared memory, fixed once the firmware is built. +pub(super) struct Views { + pub(super) compute: [RingView; PRIORITIES], + pub(super) tiling: [RingView; PRIORITIES], + pub(super) fragment: [RingView; PRIORITIES], + /// Completion stamp page A: one word per QID, written by the firmware. + pub(super) stamps: *const AtomicU32, + pub(super) qos: *mut u8, + pub(super) recovery: *const AtomicU32, + pub(super) mailbox: *const super::Mailbox, +} + +// SAFETY: The views only name memory and a mailbox owned by the firmware session, which outlives +// every user of the port. Accesses are atomic words or serialized by the port's locks. +unsafe impl Send for Views {} +// SAFETY: As above. +unsafe impl Sync for Views {} + +#[pin_data] +pub(super) struct Port { + views: SetOnce, + /// Register access for tracing the core power state at the doorbell. + regs: SetOnce, + /// Compute work-ring slots reserved by publishers that have not exposed them yet. + #[pin] + rings: SpinLock<[u32; PRIORITIES]>, + #[pin] + qos: SpinLock, + /// Serializes every render publication, with or without the device mutex: the tiling + /// and fragment work rings have one host producer at a time, and same-VM attachment + /// overlaps are ordered by publication order (see hazard.rs). + #[pin] + render: Mutex<()>, + /// The accepted recovery generation, mirrored for publishers without the device mutex. + generation: AtomicU64, + /// Device publication order shared by compute and render commands. + publications: AtomicU64, + /// Compute work slots written per priority class. + compute_written: [AtomicU64; PRIORITIES], + /// `compute_written` as read before the latest compute doorbell of each class: every slot + /// counted there was in its ring when the firmware handled that doorbell. + compute_announced: [AtomicU64; PRIORITIES], + /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. + effort_full: AtomicBool, + submissions: AtomicU64, +} + +/// Room in both render work rings of one priority, observed under the render publication +/// lock. While that lock is held no other host producer moves, and the firmware consumers only +/// advance, so the observed producers stay current and the room stays available. +#[derive(Clone, Copy)] +pub(super) struct RenderRoom { + priority: u8, + /// Fragment, then tiling producer. + producers: [u32; 2], +} + +impl RenderRoom { + pub(super) fn priority(&self) -> u8 { + self.priority + } +} + +/// A compute work-ring slot reserved before any command becomes visible. Dropping an unused +/// claim returns the slot. +pub(super) struct OuterClaim<'a> { + port: &'a Port, + priority: usize, + armed: bool, +} + +impl Drop for OuterClaim<'_> { + fn drop(&mut self) { + if self.armed { + let mut reserved = self.port.rings.lock(); + reserved[self.priority] = reserved[self.priority].saturating_sub(1); + } + } +} + +impl Port { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + views: SetOnce::new(), + regs: SetOnce::new(), + rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), + qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + render <- new_mutex!((), "g17::Port::render"), + generation: AtomicU64::new(0), + publications: AtomicU64::new(0), + compute_written: [const { AtomicU64::new(0) }; PRIORITIES], + compute_announced: [const { AtomicU64::new(0) }; PRIORITIES], + effort_full: AtomicBool::new(false), + submissions: AtomicU64::new(0), + }), + GFP_KERNEL, + ) + } + + /// Called once while the session is built, before any publisher can see the port. + pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { + if self.views.populate(views) && self.regs.populate(regs) { + Ok(()) + } else { + Err(EBUSY) + } + } + + fn views(&self) -> Result<&Views> { + self.views.as_ref().ok_or(ENODEV) + } + + fn ring(&self, priority: u8) -> Result<(usize, RingView)> { + let index = usize::from(priority); + Ok((index, *self.views()?.compute.get(index).ok_or(EINVAL)?)) + } + + /// Reserves one slot of the compute work ring of `priority`. EAGAIN while firmware has not + /// consumed enough slots; only consumption, never a waiting publisher, frees one. + pub(super) fn reserve_compute(&self, priority: u8) -> Result> { + let (index, ring) = self.ring(priority)?; + let mut reserved = self.rings.lock(); + if ring.free()? <= reserved[index] { + return Err(EAGAIN); + } + reserved[index] += 1; + Ok(OuterClaim { + port: self, + priority: index, + armed: true, + }) + } + + /// Whether a compute publication at `priority` can reserve a slot now. + pub(super) fn compute_ready(&self, priority: u8) -> Result { + let (index, ring) = self.ring(priority)?; + let reserved = self.rings.lock(); + Ok(ring.free()? > reserved[index]) + } + + /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns + /// the new producer, the retained consumer witness for this slot. + pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { + return Err(EINVAL); + } + let ring = self.views()?.compute[claim.priority]; + let mut reserved = self.rings.lock(); + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EIO); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records, checked when the session built + // it; `producer` is in range. The reservation proves firmware does not own this slot and + // the ring lock excludes every other host writer. The record is a padding-free ABI value. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + ring.slots.add(producer as usize * core::mem::size_of::()), + core::mem::size_of::(), + ); + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; + fence(Ordering::SeqCst); + reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + self.compute_written[claim.priority].fetch_add(1, Ordering::AcqRel); + claim.armed = false; + Ok(next) + } + + /// Compute work slots written at `priority` so far. + pub(super) fn compute_written(&self, priority: u8) -> u64 { + self.compute_written + .get(usize::from(priority)) + .map_or(u64::MAX, |written| written.load(Ordering::Acquire)) + } + + /// Compute work slots at `priority` that a doorbell has announced. + pub(super) fn compute_announced(&self, priority: u8) -> u64 { + self.compute_announced + .get(usize::from(priority)) + .map_or(0, |announced| announced.load(Ordering::Acquire)) + } + + /// Rings the compute doorbell `message` of `priority`, after noting which slots it + /// announces. + pub(super) fn notify_compute(&self, priority: u8, message: u64) -> Result { + let index = usize::from(priority); + let written = self.compute_written.get(index).ok_or(EINVAL)?.load(Ordering::Acquire); + self.notify(message)?; + self.compute_announced[index].fetch_max(written, Ordering::AcqRel); + Ok(()) + } + + /// Taken around every render publication; see `render`. + pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { + self.render.lock() + } + + fn render_rings(&self, priority: u8) -> Result<(RingView, RingView)> { + let views = self.views()?; + let index = usize::from(priority); + Ok(( + *views.fragment.get(index).ok_or(EINVAL)?, + *views.tiling.get(index).ok_or(EINVAL)?, + )) + } + + /// Reads both render work rings of `priority` once. Returns their producers if both have a + /// free slot; the room holds for as long as `guard`. Only firmware consumption, never a + /// publisher without the render lock, changes it. + pub(super) fn observe_render_room( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + ) -> Result> { + let (fragment, tiling) = self.render_rings(priority)?; + let cursors = [fragment.cursors()?, tiling.cursors()?]; + let room = |[first, second, producer]: [u32; 3]| { + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + room(first).min(room(second)) + }; + Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + })) + } + + /// Copies a render pair's fragment, then tiling record into the slots after the producers + /// that `room` observed, without exposing them, as `InitData::publish_work_pair` does for a + /// deferred pair. Returns the producers that expose them. + pub(super) fn write_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + room: &RenderRoom, + fragment: &WorkSlot, + tiling: &WorkSlot, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + let slot = |ring: &RingView, producer: u32| -> Result<(*mut u8, u32)> { + if producer >= WORK_RING_SLOTS { + return Err(EINVAL); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. + let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + Ok((slot, (producer + 1) % WORK_RING_SLOTS)) + }; + let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; + let (tiling_slot, tiling_next) = slot(&tiling_ring, room.producers[1])?; + let copy = |record: &WorkSlot, destination: *mut u8| { + // SAFETY: The room proves that the slot after the producer is host-owned until the + // producer moves past it, and the render lock excludes every other host writer. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + destination, + core::mem::size_of::(), + ) + } + }; + copy(fragment, fragment_slot); + commit_accounting()?; + fence(Ordering::SeqCst); + publish_late_tiling(); + copy(tiling, tiling_slot); + fence(Ordering::SeqCst); + Ok([fragment_next, tiling_next]) + } + + /// Exposes a pair written by `write_render_pair` with `room`, fragment first, as + /// `InitData::publish_retained_work_pair` does: `next` must follow the observed producers. + pub(super) fn expose_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + room: &RenderRoom, + next: [u32; 2], + ) -> Result { + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + for (producer, next) in room.producers.into_iter().zip(next) { + if next != (producer + 1) % WORK_RING_SLOTS { + return Err(EBUSY); + } + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*fragment_ring.producer).store(next[0], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + // SAFETY: As above. + unsafe { (*tiling_ring.producer).store(next[1], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The completion stamp of `qid` in page A. + pub(super) fn stamp(&self, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let base = self.views()?.stamps; + // SAFETY: Page A holds one word per QID up to QID_MAX; the session retains it. + let value = unsafe { (*base.add(usize::from(qid))).load(Ordering::Relaxed) }; + fence(Ordering::Acquire); + Ok(value) + } + + fn qos_view(&self) -> Result> { + // SAFETY: The session retains the QoS table for at least as long as this port. + unsafe { qos::View::from_raw(self.views()?.qos) } + } + + pub(super) fn qos_publish( + &self, + owner: qos::Owner, + scheduler: u64, + policy: Policy, + ) -> Result { + let view = self.qos_view()?; + self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + } + + /// Publishes the two owners of a render command, fragment then tiling. + pub(super) fn qos_publish_pair( + &self, + owners: [qos::Owner; 2], + scheduler: u64, + policy: Policy, + ) -> Result<[qos::Publication; 2]> { + let view = self.qos_view()?; + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + } + + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { + let view = self.qos_view()?; + self.qos.lock().cancel(&view, publication) + } + + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { + let view = self.qos_view()?; + self.qos.lock().complete(&view, owners) + } + + /// Rings the primary coprocessor's doorbell. + pub(super) fn notify(&self, message: u64) -> Result { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. + let mailbox = unsafe { &*self.views()?.mailbox }; + super::trace::doorbell(super::trace::COMPUTE, || { + self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) + }); + super::trace::message(super::trace::MAILBOX, message); + mailbox.send(super::EP_DOORBELL, message) + } + + pub(super) fn recovery_state(&self) -> Result { + let word = self.views()?.recovery; + fence(Ordering::Acquire); + // SAFETY: As for `RingView::cursors`. + Ok(unsafe { (*word).load(Ordering::Relaxed) }) + } + + pub(super) fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Called with the device mutex held after the accepted generation changed, before + /// publication reopens. + pub(super) fn set_generation(&self, generation: u64) { + self.generation.store(generation, Ordering::Release); + } + + /// Unique reservation order shared by both engines. Concurrent publishers may expose + /// their work in a different order; queue serialization and visibility use other locks. + /// The watchdog uses this number only to choose bounded cross-VM timeout grace. + pub(super) fn next_publication(&self) -> u64 { + self.publications.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// A publisher without the device mutex counts itself before it checks that full effort is + /// applied. The idle clear withdraws the flag before it rechecks the count, so either the + /// clear sees this submission or this submission sees the flag withdrawn. + pub(super) fn note_submission(&self) -> bool { + self.submissions.fetch_add(1, Ordering::SeqCst); + self.effort_full.load(Ordering::SeqCst) + } + + pub(super) fn submissions(&self) -> u64 { + self.submissions.load(Ordering::SeqCst) + } + + pub(super) fn set_effort_full(&self, full: bool) { + self.effort_full.store(full, Ordering::SeqCst); + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 96b363bc27447e..b0e21e70bbe18e 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,8 +3,18 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. +use core::sync::atomic::AtomicU32; + pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; + +/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after +/// the last job. Not measured on other firmware; at or below it the hold stays off. +pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; + +/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU +/// cores powered until the GPU has been idle this long ([`Hold`]). +pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { @@ -17,6 +27,7 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, + submissions: u64, } impl Effort { @@ -25,12 +36,19 @@ impl Effort { streaming: false, idle_since: None, applied: 0, + submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; + self.submissions = self.submissions.wrapping_add(1); + } + + /// Count of published submissions, for [`Hold::step`]. + pub(super) fn submissions(&self) -> u64 { + self.submissions } pub(super) fn applied(&self) -> u32 { @@ -62,6 +80,83 @@ impl Effort { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum HoldAction { + None, + /// Send the keep-powered idle policy. + Assert, + /// Send the power-off idle policy again. + Release, + Recheck(u64), +} + +/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job +/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer +/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off +/// policy once no work has been submitted or in flight for that long. Checked from the event +/// worker, which runs on every completion, well inside the firmware's 2 ms. +pub(super) struct Hold { + active: bool, + seen: u64, + since: Option, +} + +impl Hold { + pub(super) const fn new() -> Self { + Self { + active: false, + seen: 0, + since: None, + } + } + + pub(super) fn active(&self) -> bool { + self.active + } + + pub(super) fn step( + &mut self, + now_ns: u64, + in_flight: bool, + submissions: u64, + hold_ms: u32, + ) -> HoldAction { + let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; + let fresh = submissions != self.seen; + self.seen = submissions; + if in_flight || fresh { + self.since = None; + } + if !self.active { + return if enabled && (in_flight || fresh) { + HoldAction::Assert + } else { + HoldAction::None + }; + } + if in_flight { + return HoldAction::None; + } + let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; + let since = *self.since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= hold_ns { + HoldAction::Release + } else { + HoldAction::Recheck(hold_ns - idle) + } + } + + pub(super) fn asserted(&mut self) { + self.active = true; + } + + pub(super) fn released(&mut self) { + self.active = false; + self.since = None; + } +} + /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -137,3 +232,31 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] +mod tests { + use super::*; + + #[test] + fn holds_until_idle_for_the_delay_then_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); + // A new submission restarts the idle period. + assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + + #[test] + fn the_default_delay_never_holds_and_lowering_it_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); + assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 8563beda979e66..eb2366fa3451b2 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -20,6 +20,34 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; +/// One ready packet's monotonic compute-admission budget. The callback checks +/// packet cancellation, logical close, VM status and device loss even when no +/// capacity/recovery event arrives. Wakes and nested constructors never renew it. +pub(super) struct Admission<'a> { + deadline: u64, + check: &'a dyn Fn() -> Result, +} + +impl<'a> Admission<'a> { + pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { + Self { deadline, check } + } + + pub(super) fn check(&self) -> Result { + (self.check)()?; + if super::now_ns() >= self.deadline { return Err(EBUSY); } + Ok(()) + } + + /// Ignore condvar remaining-time reports: only the monotonic deadline counts. + pub(super) fn slice_ms(&self) -> Result { + self.check()?; + let remaining = self.deadline.saturating_sub(super::now_ns()); + if remaining == 0 { return Err(EBUSY); } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) + } +} + #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -27,20 +55,37 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { + #[cfg(CONFIG_KUNIT)] pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -92,6 +137,27 @@ impl Gate { } } + /// Packet admission may wait for recovery only within its original budget. + /// Called with no device mutex or existing preparation lease. + pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let _ = self.changed.wait_timeout( + &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); + } + } + fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -113,20 +179,66 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); + } + + /// Runtime failure/recovery must not join an indefinitely stalled builder. + /// Timeout retains every graph and reservation; it never authorizes freeing. + pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + if super::now_ns() >= deadline { return Err(ETIMEDOUT); } + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); + } + Ok(()) } /// Join builders with no device, queue or mapping lock held. @@ -139,7 +251,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -153,6 +265,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, @@ -184,3 +298,101 @@ impl Drop for Lease { } } } + +#[kunit_tests(asahi_g17_growth_gate)] +mod growth_gate_tests { + use super::*; + #[test] + fn recovery_can_join_between_growth_admissions() { + let gate = Gate::new().unwrap(); + let outer = gate.try_enter().unwrap(); + // The growth handoff must drop this lease before enter can wait. + drop(outer); + gate.close(); + assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); + gate.wait_drained(); + assert!(gate.reopen()); + let growth = gate.enter(false).unwrap(); + assert!(growth.is_current()); + drop(growth); + } +} + +#[kunit_tests(asahi_g17_compute_deadline)] +mod compute_deadline_tests { + use super::*; + + #[test] + fn spent_budget_refuses_even_an_open_gate() { + let gate = Gate::new().unwrap(); + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn closed_gate_expires_without_any_recovery_wake() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(super::super::now_ns() - start < 500_000_000); + gate.wait_drained(); + assert!(gate.reopen()); + } + + #[test] + fn cancellation_without_capacity_or_recovery_events() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || { + if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }; + let admission = Admission::new(start + 2_000_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn recovery_and_binding_retry_churn_cannot_renew_budget() { + let gate = Gate::new().unwrap(); + let deadline = super::super::now_ns() + 20_000_000; + let check = || Ok(()); + let admission = Admission::new(deadline, &check); + loop { + // Re-entry models a channel taken between admission/publication. + // Recovery joins each dropped lease, then wakes the same budget. + gate.close(); + gate.changed.notify_all(); + gate.wait_drained(); + assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); + match gate.enter_until(&admission) { + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } + Err(error) => { assert_eq!(error, EBUSY); break; } + } + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn packet_status_and_cancel_precede_available_capacity() { + let gate = Gate::new().unwrap(); + for error in [ECANCELED, EIO] { + let check = || Err(error); + let admission = Admission::new(u64::MAX, &check); + assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 9a100db4cffdfc..b36656b0174a7b 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -2,17 +2,14 @@ //! Shared QoS publication and exact-completion accounting for every engine. //! -//! The device publication lock serializes this host state. Publishing identity +//! The port's QoS lock serializes this host state. Publishing identity //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{ - fw::{qos as fw, queue::Policy}, - object::KernelObject, -}; +use super::fw::{qos as fw, queue::Policy}; use crate::hw::t8140::qos as cfg; use core::{ - mem::{offset_of, size_of}, + mem::offset_of, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; @@ -30,15 +27,17 @@ pub(crate) struct View<'a> { times_and_counts: &'a [AtomicU64; 7], } impl<'a> View<'a> { - pub(crate) fn new(object: &'a KernelObject) -> Result { - let ptr = object.pointer(0, size_of::())?; + /// # Safety + /// + /// `ptr` must span a complete QoS table that stays mapped for `'a`. + pub(crate) unsafe fn from_raw(ptr: *mut u8) -> Result { if ptr as usize & 7 != 0 { return Err(EINVAL); } // SAFETY: pointer spans the complete layout, whose asserted offsets // and size bound every array below. Integer atomics have their integer - // representation and alignment. The retained object borrow keeps every - // shared reference alive; accesses use each field's fixed width. + // representation and alignment. The caller keeps every shared + // reference alive; accesses use each field's fixed width. Ok(unsafe { Self { rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), @@ -54,6 +53,14 @@ impl<'a> View<'a> { fn row(&self, qid: usize) -> [u8; 8] { core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) } + /// The complete row in one access. The table is uncached: a publication reads + /// each row once, before it stores anything. + fn row_word(&self, qid: usize) -> [u8; 8] { + let row = core::ptr::from_ref(&self.rows[qid]).cast::(); + // SAFETY: Rows are eight bytes at eight-byte offsets of the eight-byte + // aligned table (checked in from_raw), so the row is one aligned word. + unsafe { (*row).load(Ordering::Relaxed) }.to_le_bytes() + } } #[derive(Clone, Copy, Debug)] @@ -109,66 +116,111 @@ impl Accounting { policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { - let (qid, qos, dm) = owner.indices()?; + let [publication] = self.publish_owners(view, [owner], scheduler, policy, clock)?; + Ok(publication) + } + + /// Publishes the owners of one command's queues, in order, with the same + /// firmware-visible values as one `publish` per owner. The owners use + /// distinct queues and data masters. Every check precedes the first store, + /// so a refusal changes nothing; every table word is read before the first + /// store, so no uncached read waits for an earlier store to drain. + pub(crate) fn publish_owners( + &mut self, + view: &View<'_>, + owners: [Owner; N], + scheduler: u64, + policy: Policy, + clock: impl FnOnce() -> (u64, u64), + ) -> Result<[Publication; N]> { if scheduler == 0 { return Err(EINVAL); } - let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; - let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; - let starts_busy = self.dm_in_flight[dm] == 0; - let time = if starts_busy { + let mut indices = [(0, 0, 0); N]; + for (index, owner) in owners.iter().enumerate() { + indices[index] = owner.indices()?; + let (qid, _, dm) = indices[index]; + if indices[..index] + .iter() + .any(|&(other_qid, _, other_dm)| other_qid == qid || other_dm == dm) + { + return Err(EINVAL); + } + self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + } + let rows = indices.map(|(qid, _, _)| view.row_word(qid)); + let queue_submitted = indices.map(|(qid, _, _)| view.queue_submitted[qid].load(Ordering::Relaxed)); + let owner_submitted = indices.map(|(_, qos, _)| view.owner_submitted[qos].load(Ordering::Relaxed)); + let dm_submitted = indices.map(|(_, _, dm)| view.times_and_counts[3 + dm].load(Ordering::Relaxed)); + let starts_busy = indices.map(|(_, _, dm)| self.dm_in_flight[dm] == 0); + let busy = starts_busy.iter().any(|&starts| starts); + let intervals = if busy { view.intervals.load(Ordering::Relaxed) } else { 0 }; + for (index, owner) in owners.iter().enumerate() { + let (qid, _, _) = indices[index]; + if self.in_flight[qid] != 0 + && (rows[index][0] != owner.qos || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + } + let time = if busy { let (counter, frequency) = clock(); time_from_host(counter, frequency)? } else { 0 }; - if self.in_flight[qid] != 0 - && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master) - { - return Err(EBUSY); + let class = policy.qos_class(); + let mut first = false; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, _) = indices[index]; + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(class, Ordering::Relaxed); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); + first |= self.in_flight[qid] == 0; } - view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); - let record = view.row(qid); - view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(policy.qos_share(), Ordering::Relaxed); - if self.in_flight[qid] == 0 { + if first { fence(Ordering::SeqCst); } - view.queue_submitted[qid].store( - view.queue_submitted[qid] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - view.owner_submitted[qos].store( - view.owner_submitted[qos] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - if starts_busy { - view.times_and_counts[dm].store(time, Ordering::Relaxed); - } - let submitted = &view.times_and_counts[3 + dm]; - let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); - submitted.store(dm_submitted, Ordering::Relaxed); - if starts_busy { - view.intervals.store( - view.intervals.load(Ordering::Relaxed).wrapping_add(1), + let mut publications = [Publication { + record: [0; 8], + owner: owners[0], + dm_submitted: 0, + }; N]; + let mut intervals = intervals; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, dm) = indices[index]; + view.queue_submitted[qid].store(queue_submitted[index].wrapping_add(1), Ordering::Relaxed); + // Owners of one logical queue share their submission count. + let earlier = indices[..index].iter().filter(|&&(_, other, _)| other == qos).count(); + view.owner_submitted[qos].store( + owner_submitted[index].wrapping_add(earlier as u32 + 1), Ordering::Relaxed, ); + if starts_busy[index] { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = dm_submitted[index].wrapping_add(1); + view.times_and_counts[3 + dm].store(submitted, Ordering::Relaxed); + if starts_busy[index] { + intervals = intervals.wrapping_add(1); + view.intervals.store(intervals, Ordering::Relaxed); + } + let mut record = rows[index]; + record[0] = owner.qos; + record[1] = class; + self.in_flight[qid] += 1; + self.dm_in_flight[dm] += 1; + self.dm_by_qid[qid] = owner.data_master; + publications[index] = Publication { + record, + owner: *owner, + dm_submitted: submitted, + }; } - self.in_flight[qid] = next_qid; - self.dm_in_flight[dm] = next_dm; - self.dm_by_qid[qid] = owner.data_master; fence(Ordering::SeqCst); - Ok(Publication { - record, - owner, - dm_submitted, - }) + Ok(publications) } /// A possibly exposed producer must retain its reference until completion. diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f6..e1e7552c243f24 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,16 +44,32 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; +/// A wake never renews admission's budget. Short observations also notice a +/// packet cancellation that has no corresponding pair-release notification. +fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { + let remaining = deadline_ns.saturating_sub(now_ns); + if remaining == 0 { + return Err(EBUSY); + } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) +} + struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, + /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail + /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. + spared: u64, kill_hold: u64, released: u64, epoch: u64, @@ -68,6 +85,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, + spared: 0, kill_hold: 0, released: 0, epoch: 0, @@ -92,7 +110,7 @@ impl State { & Self::MASK == 0 => { - Err(ENOSPC) + Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) } _ => Ok(()), } @@ -115,18 +133,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,8 +195,51 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; + } + } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -188,9 +251,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -213,6 +298,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -251,6 +358,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -258,6 +366,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -283,23 +397,55 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { - pub(crate) fn new() -> Result> { + #[cfg(CONFIG_KUNIT)] + pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + renders: Option>, + ) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, + waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); + if count >= self.max_queues { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( count, @@ -316,16 +462,22 @@ impl Pool { self: &Arc, owner: u64, request: Option>, + deadline_ns: u64, + check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let started = crate::g17::now_ns(); + let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); + let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); let mut expired = false; let mut ticket = None; let result = (|| loop { + check()?; + let now = crate::g17::now_ns(); + let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -337,20 +489,26 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { - return Err(EIO); + if now >= quarantine_deadline { + // Behind an innocent pair's quarantine (another VM wedged the GPU) only this + // publication fails, not the VM: it recovers once the pair drains. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + return Err(if spared { EBUSY } else { EIO }); + } + let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { + return Err(ERESTARTSYS); } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; continue; } state.refusal(owner)?; + expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -376,18 +534,17 @@ impl Pool { } } } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { + } + // Keep the established hazard wait/fallback rule, but cap every + // sleep with the caller's original deadline and cancellation poll. + let slice = if plan != Plan::Any { + remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) + } else { + remaining_ms + }; + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { return Err(ERESTARTSYS); } })(); @@ -420,8 +577,51 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -431,7 +631,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -446,9 +646,79 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; + state.spared &= !released; state.kill_hold = 0; drop(state); - self.released.notify_all(); + self.returned(); + } +} + +#[kunit_tests(asahi_g17_admission_accounting)] +mod accounting_tests { + use super::*; + + #[test] + fn every_logical_owner_fits_without_speculative_spares() { + let mut state = State::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot = state.acquire(owner).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); + state.release(slot); + } + assert_eq!(state.active, 0); + assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); + } + + #[test] + fn close_does_not_reuse_an_active_lease() { + let mut state = State::new(); + let slot = state.acquire(1).unwrap(); + state.released |= 1u64 << slot; + assert!(!state.reassign_unused_spare(slot, Some(1), None)); + state.release(slot); + assert!(state.reassign_unused_spare(slot, Some(1), None)); + assert_eq!(state.owners[usize::from(slot)], 0); + assert_eq!(state.refs[usize::from(slot)], 0); + assert_eq!(state.active & (1u64 << slot), 0); + } + + #[test] + fn warming_quarantine_and_exit_hold_exclude_reassignment() { + let mut state = State::new(); + let primary = state.acquire(1).unwrap(); + // A retained legacy sibling still obeys all quarantine/kill guards. + state.owners[2] = 1; + let spare = state.borrow_warm_sibling(1, primary).unwrap(); + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.release(spare); + state.quarantined |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.quarantined = 0; + state.kill_hold |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.kill_hold = 0; + assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); + assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + state.release(primary); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + } + + #[test] + fn logical_queue_count_returns_on_drop() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..MAX_EXECUTION_CONTEXTS { + leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); + assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); + drop(leases.pop()); + let replacement = pool.reserve_queue().unwrap(); + drop(leases); + assert_eq!(pool.queues.load(Ordering::Acquire), 1); + drop(replacement); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); } } @@ -456,9 +726,18 @@ impl Pool { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } @@ -468,36 +747,118 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } - pub(crate) fn quarantine(&self) { - self.pool.state.lock().quarantined |= 1u64 << self.slot; - self.pool.released.notify_all(); + /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine + /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. + pub(crate) fn quarantine(&self, spared: bool) { + let bit = 1u64 << self.slot; + let mut state = self.pool.state.lock(); + if !spared { + state.spared &= !bit; + } else if state.quarantined & bit == 0 { + state.spared |= bit; + } + state.quarantined |= bit; + drop(state); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - self.pool.state.lock().quarantined &= !(1u64 << self.slot); + let mut state = self.pool.state.lock(); + state.quarantined &= !(1u64 << self.slot); + state.spared &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } @@ -536,6 +897,172 @@ impl Pool { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); + } +} + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; + let state = self.state.lock(); + let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { + bits | if *owner != 0 { 1u64 << slot } else { 0 } + }) & State::MASK; + let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; + writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", + used.count_ones(), State::MASK.count_ones(), retained.count_ones(), + (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), + (state.kill_hold & State::MASK).count_ones()) + } +} + +#[kunit_tests(asahi_g17_fault_admission)] +mod fault_capacity_tests { + use super::*; + + #[test] + fn a_guilty_failure_makes_a_spared_quarantine_guilty() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + lease.quarantine(true); + assert_ne!(pool.state.lock().spared & bit, 0); + lease.quarantine(false); + assert_eq!(pool.state.lock().spared & bit, 0); + // Once guilty, a later spared failure does not make it spared again. + lease.quarantine(true); + assert_eq!(pool.state.lock().spared & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + pool.release_owner(1); + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.quarantined | state.spared, 0); + } + + #[test] + fn closed_faulting_clients_return_leases_after_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + for owner in 1..=512 { + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + lease.quarantine(false); + pool.release_owner(owner); + let bit = 1u64 << lease.slot(); + assert_ne!(pool.state.lock().active & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + // The runtime, not an error fence, supplies this exact drain witness. + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.active, 0); + assert_eq!(state.quarantined, 0); + assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } + } + + #[test] + fn published_exit_kill_holds_survive_close_and_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); + // A failed (unpublished) control may roll back its hold. Published kills + // keep the existing processor-stop rule; this test must not weaken it. + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().kill_hold, 0); + } +} + +impl Pool { + pub(crate) fn owner_active(&self, owner: u64) -> bool { + let state = self.state.lock(); + state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) + } +} + +#[kunit_tests(asahi_g17_shared_render_depth)] +mod shared_depth_tests { + use super::*; + #[test] + fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { + let mut state = State::new(); + let first = state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } + assert_eq!(state.acquire(1), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + state.release(first); + assert_eq!(state.acquire(1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_admission_deadline)] +mod deadline_tests { + use super::*; + + #[test] + fn all_pairs_busy_no_progress_expires_without_release() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + // No pair has room, including the requesting owner's retained pair. + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + assert!(matches!(result, Err(error) if error == EBUSY)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn busy_pair_observes_cancellation_without_a_release_event() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }); + assert!(matches!(result, Err(error) if error == ECANCELED)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn cancellation_precedes_available_capacity() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + Err(error) if error == ECANCELED)); + assert_eq!(pool.state.lock().active, 0); + } + + #[test] + fn unrelated_wake_storm_never_renews_absolute_deadline() { + let deadline = 2_000_000_000; + // Each wake is allowed to report the full jiffy budget remaining. Only + // monotonic time, not that report or a wake count, bounds admission. + for now in (0..deadline).step_by(100_000) { + assert!(wait_slice_ms(deadline, now).unwrap() <= 10); + } + assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); + assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb7609..9c4c5c92d049df 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,8 +9,13 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, - object::{Allocator, CpuMap, KernelObject}, + fw::{ + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, + }, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -25,10 +30,20 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -78,24 +93,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -107,47 +130,80 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new( + alloc: &Allocator<'_>, + objects: &Arc, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -158,8 +214,11 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, + operand_state, }) } @@ -221,9 +280,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -234,12 +293,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -258,7 +317,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); + return Err(EAGAIN); } } Ok(()) @@ -276,7 +335,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(CURSORS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -308,6 +369,10 @@ use crate::g17::{ }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; +/// Position in its queue's active ring from which a publication withholds its doorbell: +/// with two unretired commands ahead, it cannot be the next to start. +const WITHHOLD_DEPTH: usize = 3; + struct Fifo { entries: [Option; IN_FLIGHT], head: usize, @@ -389,11 +454,21 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous { - binding: Option, - context: Arc, +struct Previous> { + binding: Option, + context: C, publication: Option<(u64, u64)>, } +impl Previous { + /// The exact drain witness covers the replacement publication even when + /// its public fence failed. An unpublished replacement still needs rollback. + fn retire_witnessed(previous: &mut Option) { + if previous.as_ref().is_some_and(|p| p.publication.is_some()) { + drop(previous.take()); + } + } +} + /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -453,12 +528,21 @@ pub(crate) enum PoolPhase { Vacant, } +/// Builder ownership of one physical release transaction. The preparation +/// lease pins this registry until abort/commit; owner close may remove lookups. +#[derive(Clone, Copy)] +pub(crate) struct Growth { + qid: u8, + phase: PoolPhase, +} + pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -471,6 +555,8 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, + /// This binding has never crossed the successor visibility boundary. + unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -484,8 +570,14 @@ pub(crate) struct Queue { /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -499,27 +591,86 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, + /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more + /// unretired commands of this queue ahead of them, the publication of the first, and the + /// compute work slots written at its priority once its slot was. + withheld_doorbell: Option<(u64, u64, u64)>, } impl Queue { + /// Returns the withheld doorbell once at most one command of this queue that precedes the + /// first publication it covers has no completion record yet. A doorbell of the same class + /// rung after the last withheld work slot was written announced every one of them: nothing + /// is left to ring. `announced` gives the slots announced per class. + /// + /// Retirement waiting for a stamp does not hold the doorbell back, and any departure from + /// ordinary retirement (timeout, kill, quarantine, replay, recovery or exit) rings it at + /// once: the firmware then sees every published command, as without withholding. + pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { + let (doorbell, first, written) = self.withheld_doorbell?; + if announced(((doorbell >> 2) & 3) as u8) >= written { + self.withheld_doorbell = None; + return None; + } + let settling = self.settling(); + let incomplete = self + .active + .iter() + .filter(|active| active.published < first && active.observed.is_none()) + .count(); + if !settling && incomplete > 1 { + return None; + } + self.withheld_doorbell = None; + Some(doorbell) + } + + /// The queue has left ordinary retirement: a timeout, kill, quarantine, replay, recovery + /// witness or exit is in progress. + fn settling(&self) -> bool { + self.quarantined + || self.released + || self.exit.is_some() + || self.replays.len != 0 + || self.replay_retirement + || self.awaiting_witness + || self.retire_pending + || self.retired_by_teardown + || self.spared_quarantine + || self.failure_status_pending + || self.terminal.is_some() + } + + /// Takes the withheld doorbell regardless of its predecessors. + pub(crate) fn take_withheld_doorbell(&mut self) -> Option { + self.withheld_doorbell.take().map(|(doorbell, _, _)| doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -527,6 +678,7 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, + unpublished_binding: true, kick, pool, graph, @@ -539,8 +691,11 @@ impl Queue { installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, + claims: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -554,6 +709,9 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), + withheld_doorbell: None, }) } pub(crate) fn qid(&self) -> u8 { @@ -566,6 +724,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -574,18 +747,16 @@ impl Queue { pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - pub(crate) fn room(&self) -> bool { - !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT - } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: Arc::as_ptr(active.packet.completion.status()) as usize, + vm: active.packet.context.vm().id(), published: active.published, finished: active.observed.is_some(), + failed: self.quarantined, }) } @@ -603,6 +774,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -626,38 +818,56 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + + pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { + self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + } + + /// A closed owner can still retain its last commands or their completion + /// stamps. Admission may wait for retirement, without treating that close + /// as permission to reuse any firmware-owned backing. + pub(crate) fn release_settling(&self) -> bool { + // release_owner keeps the old owner while its last commands retire. + // Waiting only once owner is None misses precisely that interval. + release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { @@ -780,9 +990,7 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = front - .observed_at - .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); + let record_aged = retirement::record_aged(front.observed_at, now_ns()); let timestamps = retirement::poll( &ticket, front.observed, @@ -814,6 +1022,11 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); + crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, + ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -829,6 +1042,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { @@ -929,6 +1143,10 @@ impl Queue { self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; + // The stamp and fully consumed ring proved the replacement retired. + // Keeping its predecessor only on the fault path prevents recycling + // forever: neither another retirement nor an exit kill can release it. + Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -937,15 +1155,9 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } if self.released { self.owner = None; + self.released_at = now_ns(); } } Ok(()) @@ -1027,6 +1239,128 @@ impl Queue { } } +/// Validate the complete unpublished item budget before changing descriptors, +/// work counts or queue producers. Leave one item empty across ring wrap. +fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { + if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); + return Err(EAGAIN); + } + Ok(()) +} + +/// Retained commands are capacity, not a failed VM. Nothing has been written +/// when this check runs, so the outer publisher can wait for retirement under +/// its original admission deadline, just as it does for a full firmware ring. +fn retained_capacity( + quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, +) -> Result { + if quarantined || released { + return Err(EBUSY); + } + if active >= IN_FLIGHT || (replays != 0 && !replay_front) { + return Err(EAGAIN); + } + Ok(()) +} + +/// Logical close promises a future healthy handoff, not immediate reuse. +/// Exact retirement still owns the transition. +fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { + released && !quarantined && (active != 0 || retiring) +} + +#[kunit_tests(asahi_g17_compute_admission)] +mod admission_tests { + use super::*; + + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1)); + } + + #[test] + fn retained_ring_pressure_waits_without_discarding_work() { + let mut active = Fifo::new(); + for sequence in 0..IN_FLIGHT { + active.push(sequence).unwrap(); + } + assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!(active.front(), Some(&0)); + assert_eq!(active.len, IN_FLIGHT); + assert_eq!(active.pop(), Some(0)); + assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!(active.front(), Some(&1)); + } + + #[test] + fn replay_front_and_terminal_ownership_remain_distinct() { + assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); + assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); + assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + for (quarantined, released) in [(true, false), (false, true), (true, true)] { + assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + } + } + + #[test] + fn item_refusal_is_retryable_and_preserves_the_empty_slot() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + } + + #[test] + fn bad_item_cursors_are_not_backpressure() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); + assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + } + + #[test] + fn closed_owner_waits_for_its_last_command() { + assert!(release_pending(true, false, 1, false)); + assert!(release_pending(true, false, 0, true)); + assert!(!release_pending(true, false, 0, false)); + } + + #[test] + fn live_or_quarantined_owners_do_not_promise_capacity() { + assert!(!release_pending(false, false, 1, true)); + assert!(!release_pending(true, true, 1, true)); + } +} + impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1062,11 +1396,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1106,7 +1444,11 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), + if same_vm { 2 } else { 1 }); self.owner = Some(owner); + self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1115,6 +1457,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1134,6 +1479,13 @@ impl Queue { } self.retirement_ready = true; } + // Rebinding an already detached queue has no Previous. If its + // first command is rejected or cancelled before publication, the + // empty active FIFO is still the original idle witness. No future + // completion can turn retirement_ready back on for this owner. + if self.previous.is_none() { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1162,7 +1514,9 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1220,6 +1574,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1259,6 +1614,7 @@ impl Queue { ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); + self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1283,7 +1639,7 @@ impl Queue { let pages = self.pool.take_pages(); let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; + self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1298,12 +1654,12 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } + if !self.can_rearm() { return Err(EAGAIN); } + self.withheld_doorbell = None; self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1314,11 +1670,7 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { - if blocks < self.growth_blocks { return Err(EAGAIN); } - self.growth_blocks = blocks; - return Ok(false); - } + if self.growing { return Err(EAGAIN); } if self.quarantined || self.released || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) @@ -1327,7 +1679,8 @@ impl Queue { let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - } else { self.pool_phase = PoolPhase::Vacant; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) @@ -1344,16 +1697,35 @@ impl Queue { return Err(EAGAIN); } } + if alias.is_none() { return Err(EINVAL); } + let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - let old_alias = binding.pool_alias.take(); - let old_pages = self.pool.take_pages(); - self.pool.rearm(pages.take().ok_or(EINVAL)?)?; - init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; - binding.pool_alias = alias.take(); + // No ownership moves before every fallible descriptor/header operation. + // The old release is consumed; no queue can use this row until commit. + init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + self.pool.replace_pages(pages)?; + let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; + self.growing = false; + self.growth_blocks = 0; + Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + } + + pub(crate) fn growth_token(&self) -> Option { + self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + } + + /// Transfer an abandoned builder's release back to the event worker. Do not + /// revoke the published release or free backing here: vacancy still requires + /// its exact control-consumer witness. A different transaction is untouched. + pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { + if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { + return Ok(None); + } self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + self.release_owner().map(Some) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -1436,14 +1808,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1495,6 +1870,15 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1504,12 +1888,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - (&mut self.records, slot * RECORD_STRIDE + offset) + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) + } else { + (&mut self.cached_records, CACHED_RECORD_STRIDE) + }; + (object, record_start(slot, offset, window, stride)?) } else { - (&mut self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1535,7 +1922,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1563,7 +1950,8 @@ pub(crate) trait Host { fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; - fn notify(&mut self, message: u64) -> Result; + fn notify_compute(&mut self, priority: u8, message: u64) -> Result; + fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; } @@ -1580,6 +1968,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1614,10 +2013,22 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { return Err(EAGAIN); } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); + item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1640,6 +2051,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -1661,6 +2073,8 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1682,6 +2096,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -1707,7 +2122,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( @@ -1724,17 +2139,19 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence); + let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, record)?; + wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; @@ -1743,6 +2160,7 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1760,7 +2178,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -1769,16 +2187,24 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - Err(error) - })?; + })().or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; } + Err(error) + })?; visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1789,14 +2215,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -1826,10 +2254,12 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -1854,13 +2284,34 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // A publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. + // + // A doorbell makes the firmware read the whole work ring of its class, + // and each one it handles while finishing a command delays that + // command's completion and the release of its waiters. A command with + // two or more unretired commands of its queue ahead cannot start soon: + // its doorbell is withheld until at most one precedes it (see + // `due_doorbell`), and any doorbell rung meanwhile on the same ring + // announces its work slot earlier. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; + if self.active.len >= WITHHOLD_DEPTH + && !self.settling() + && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) + { + let written = host.compute_written(policy.priority()); + let entry = self.withheld_doorbell.get_or_insert((doorbell, published, written)); + entry.2 = written; + } else { + host.notify_compute(policy.priority(), doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _, _)| withheld == doorbell) { + self.withheld_doorbell = None; + } + } + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { @@ -1898,11 +2349,8 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } + retained_capacity(self.quarantined, self.released, self.active.len, + self.replays.len, replay_front)?; if !packet.context.is_current() || !self .binding @@ -1969,7 +2417,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -1982,3 +2430,236 @@ impl Queue { self.terminal.take() } } + +impl Queue { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", + self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), + u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), + u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; + match self.graph.cursors() { + Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + _ => writeln!(f, "items_unavailable=1"), + }?; + writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), + u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + if let Some(binding) = self.binding.as_ref() { + binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + } + if let Some(previous) = self.previous.as_ref() { + previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + } + Ok(()) + } +} + +#[kunit_tests(asahi_g17_fault_ownership)] +mod fault_ownership_tests { + use super::*; + use core::sync::atomic::AtomicUsize; + + struct Lease(Arc); + impl Lease { + fn new(used: &Arc) -> Self { + used.fetch_add(1, Ordering::Relaxed); + Self(used.clone()) + } + } + impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + + // This tests the production predecessor ownership operation without firmware + // storage. Runtime tests must separately establish its drain/kill witness. + fn faulting_clients(publication: Option<(u64, u64)>) { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + for _ in 0..512 { + let mut previous = Some(Previous { + binding: Some(Lease::new(&used)), + context: Lease::new(&used), + publication, + }); + assert_eq!(used.load(Ordering::Relaxed), 2); + Previous::retire_witnessed(&mut previous); + assert_eq!(previous.is_none(), publication.is_some()); + if publication.is_none() { + assert_eq!(used.load(Ordering::Relaxed), 2); + // An unpublished replacement returns its original owner on close. + drop(previous.take()); + } + assert_eq!(used.load(Ordering::Relaxed), 0); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn faulted_published_replacements_return_to_baseline() { + faulting_clients(Some((1, 7))); + } + + #[test] + fn unpublished_replacements_survive_until_rollback() { + faulting_clients(None); + } + + #[test] + fn same_vm_replacement_without_binding_returns_its_context() { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + let mut previous = Some(Previous:: { + binding: None, + context: Lease::new(&used), + publication: Some((9, 3)), + }); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } +} + +impl Queue { + /// Offer an open client's idle physical channel to another ready job. + /// Logical queue/context ownership is unchanged. The existing binding is + /// retained through the successor's exact replacement witness. + pub(crate) fn share_idle(&mut self) -> Result { + if !self.shareable() { + return Ok(false); + } + self.release_owner()?; + Ok(self.reusable()) + } + + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready + && !self.growing && !self.quarantined && !self.awaiting_witness + && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 + && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } +} + +impl Queue { + pub(crate) fn can_rearm(&self) -> bool { + self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) + } +} + +impl Queue { + /// A failed first attempt has no future completion to undo its binding. + /// Return the replaced aliases to the caller for off-lock destruction. The + /// caller validates the current preparation epoch and cancels only a true + /// result's never-published QID; installed rings and identities stay intact. + pub(crate) fn rollback_unpublished( + &mut self, owner: u64, context: &Arc, + ) -> Result<(bool, Option)> { + if self.owner != Some(owner) || !self.unpublished_binding + || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.retire_pending || self.failure_status_pending + || self.pool_phase != PoolPhase::Populated + || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + { + return Ok((false, None)); + } + if !self.installed { return Ok((true, None)); } + let mut retired = None; + if let Some(previous) = self.previous.as_ref() { + self.graph.set_owner(&previous.context)?; + } + if let Some(mut previous) = self.previous.take() { + if let Some(binding) = previous.binding.take() { + retired = self.binding.replace(binding); + } else { + self.binding.as_mut().ok_or(EIO)?.context = previous.context; + } + } + self.unpublished_binding = false; + self.retirement_ready = true; + self.released = true; + self.owner = None; + self.released_at = now_ns(); + Ok((false, retired)) + } +} + + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source(&self, source: &Arc, + status: &Arc, epoch: u64) -> bool + { + !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source)) + } + + pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, + status: &Arc, epoch: u64) -> Result> + { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { continue; } + if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } + if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { return Err(EAGAIN); } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { return Err(ECANCELED); } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some((active.published, dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), hold))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 187fd27f8e78e6..d7386e335e1671 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -220,6 +220,9 @@ impl Queue { ))?; drop(self.active.pop()); } + // The acknowledged kill and command settlement replace the ordinary + // drain witness, including a failed first publication after rebinding. + super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421f..c2b49e7b436d77 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -66,21 +66,37 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. The firmware stores the stamp after the cache -/// maintenance that follows the record; it occasionally never stores it for a command, most -/// visibly while a render of the same VM is running. -pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; +/// before it retires on the record alone. +/// +/// The firmware stores the stamp after the cache maintenance that follows the record, so the +/// stamp, not the record, proves that the command's writes are complete and that its mappings +/// may be released. While other commands of the same submit ioctl are still running, the stamp +/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms +/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as +/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail +/// its VM. +pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; + +// A command that waits out the fallback must still retire well inside the scheduler timeout. +const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); + +/// Whether a completion record latched at `observed_at` (host time, ns) has waited out +/// [`STAMP_FALLBACK_NS`] by `now`. +pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { + observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) +} static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -/// Counts a command that retired on its record without its completion stamp; logs the first and -/// every 64th. +/// Counts a command that retired on its record because its completion stamp never arrived; +/// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, + STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -90,7 +106,12 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// must be consumed. +#[expect( + clippy::too_many_arguments, + reason = "Retirement validates the ticket against independent firmware observations." +)] pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -140,35 +161,122 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, + unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 + } + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -180,63 +288,411 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) + } + + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; if count == 0 { - return Ok(()); + return Ok(!self.published_after_staging(qid, kick)); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(request.observed.is_some()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; } - let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); + } + if request.observed.is_none() { + self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) + } +} + +#[kunit_tests(asahi_g17_compute_stamp_wait)] +mod tests { + use super::*; + + const KICK: u64 = 0x12_3456_7801; + + fn ticket() -> Ticket { + Ticket { + owner: 1, + submission: 1, + context_id: 1, + qid: 28, + descriptor: 0x1000, + work_node: 0x2000, + kick: KICK, + ordinal: 1, + item_producer: 4, + item_count: 64, + epoch: 1, + } + } + + fn consumed() -> Cursors { + Cursors { + producer: 4, + consumer: 4, + count: 64, + } + } + + fn own(kick: u64) -> u32 { + (kick as u32).wrapping_shl(8) + } + + #[test] + fn stamp_covers_its_kick_and_earlier_ones() { + assert!(stamp_covers(own(KICK), KICK)); + assert!(stamp_covers(own(KICK + 1), KICK)); + assert!(!stamp_covers(own(KICK - 1), KICK)); + assert!(!stamp_covers(own(KICK) | 1, KICK)); + assert!(stamp_covers(0, 0xff_ffff)); + } + + #[test] + fn a_late_stamp_is_waited_for() { + // Under games on J700 the stamp lands up to about 4 ms after the record while the + // rest of the command's ioctl runs: the command keeps its mappings until then. + let record = Some(Observation { timestamps: [10, 20] }); + let at = 1_000_000; + assert!(!record_aged(Some(at), at + 4_000_000)); + assert!(!record_aged(Some(at), at + 10_000_000)); + let waiting = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + 10_000_000), + ); + assert_eq!(waiting, Ok(None)); + let stamped = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(stamped, Ok(Some([10, 20]))); + } + + #[test] + fn a_later_kick_stamp_retires_it() { + let record = Some(Observation { timestamps: [10, 20] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK + 3), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(result, Ok(Some([10, 20]))); + } + + #[test] + fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { + let record = Some(Observation { timestamps: [0x1111, 0x2222] }); + let at = 1_000_000; + assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); + assert!(!record_aged(None, u64::MAX)); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + STAMP_FALLBACK_NS), + ); + // The record's own timestamps are published, never synthesized ones. + assert_eq!(result, Ok(Some([0x1111, 0x2222]))); + } + + #[test] + fn an_aged_record_still_waits_for_consumed_items() { + let record = Some(Observation { timestamps: [10, 20] }); + let busy = Cursors { + producer: 4, + consumer: 2, + count: 64, + }; + let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); + assert_eq!(result, Ok(None)); + } + + #[test] + fn a_zero_duration_record_fails() { + let record = Some(Observation { timestamps: [5, 5] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::Allowed, + 0, + None, + true, + ); + assert_eq!(result, Err(ENODATA)); + } +} + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t, false).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8e..16891524e25a06 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -27,7 +27,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -56,10 +55,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +76,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,15 +91,21 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } +type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); + impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -107,6 +119,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -119,16 +132,19 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + finished: KVec::new(), failures: [None, None], _slot: slot, }) } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Admission is checked even when this queue already owns a scheduler. + // Validate every request, including an existing scheduler. Physical + // admission happens after its DRM dependencies, in backend publication. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -142,11 +158,20 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); + } // One earliest failed ordinal covers every later prefix without // retaining its packet, mappings, or every failed fence. if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { @@ -156,13 +181,17 @@ impl Queue { }); } + #[expect( + clippy::too_many_arguments, + reason = "Enqueue keeps validated command, ordering, and destination owners explicit." + )] fn enqueue( &mut self, id: u64, command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -181,6 +210,22 @@ impl Queue { let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + let sources = if engine == Engine::Compute && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[0], *sequence) + || fences.status() > 0) + && self.frontiers[1].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || fences.status() > 0) + && self.failures.iter().enumerate().all(|(index, failure)| + failure.as_ref().is_none_or(|(sequence, _)| + !Order::contains(order.wait_through[index], *sequence))) + { self.backend.input_sources(inputs) } else { None }; + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] @@ -199,7 +244,8 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), + sources.clone(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -260,7 +306,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -270,13 +316,30 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { + // An input that already signalled without error orders nothing and has no status + // left to consume (like a completed frontier producer above). Keeping it would + // send the render through the scheduler thread instead of inline publication: + // most compositor and client renders carry such already-signalled buffer fences. + if job::fence_status(fence) > 0 { + continue; + } + if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), @@ -307,7 +370,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { + ) -> Result { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -320,6 +383,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { @@ -384,9 +448,18 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes, objects)?; + let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + self.context.note_render(); + } + if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -401,6 +474,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e402..696a1486eb0ec5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -78,15 +83,18 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; + /// The next publication number, shared with compute (timeout ordering). + fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -120,6 +128,8 @@ struct Prepared { mcache: Option, } struct Active { + /// Publication number, shared with compute (crate::g17::timeout). + published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -152,8 +162,11 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -168,14 +181,31 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// A retired, never-executed group awaiting republication on this pair. - replay: Option>, + /// Retired, never-executed groups awaiting republication on this pair, in + /// their original order. Nothing else is prepared on the pair before them. + replay: Tracker>, + /// When the oldest parked group was parked or the last one republished. + replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "The packet constructor spells out the paired render firmware inputs." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -186,7 +216,9 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -211,10 +243,11 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -226,9 +259,10 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -239,8 +273,11 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, + deflake, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -251,8 +288,12 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: 0, + switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -280,20 +321,25 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.is_some() + self.active.len() != 0 || self.replay.len() != 0 } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_stalled(&self, now: u64) -> bool { + self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.clone() + self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay = None; + self.replay.pop(); + // Progress: the expiry restarts for the next parked group, if any. + self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -315,6 +361,7 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() + || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { @@ -331,17 +378,21 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -371,6 +422,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -402,11 +454,18 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -436,7 +495,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -447,7 +506,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -473,6 +532,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -503,9 +567,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -517,6 +589,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -574,10 +653,26 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -619,6 +714,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -628,6 +724,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -649,6 +746,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -658,6 +756,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -694,8 +793,10 @@ impl Pair { let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; + let published = host.next_publication()?; self.active .push(Active { + published, qos_completed: false, flist_completed: false, outer_published: false, @@ -721,18 +822,17 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -747,7 +847,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -769,6 +874,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -777,10 +885,16 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -865,6 +979,8 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, + packet); host.note_submission() })(); if let Err(error) = publish { @@ -924,6 +1040,22 @@ impl Pair { .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) }) } + /// The oldest published group still running, for cross-VM timeout attribution + /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains + /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. + pub(crate) fn timeout_head(&self) -> Option { + self.active + .iter() + .filter(|active| active.outer_published) + .find(|active| active.drained_result.is_none()) + .map(|active| crate::g17::timeout::Head { + vm: active.packet.context.vm().id(), + published: active.published, + finished: false, + failed: self.quarantined, + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -934,7 +1066,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -955,7 +1087,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -981,6 +1113,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1009,6 +1163,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1084,44 +1239,192 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - loop { - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { return Err(EIO); } - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { - return Err(EIO); - } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; + } + crate::g17::recovery::note_retired(); + crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, + &front.packet, [timestamps[0], timestamps[3]]); + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(true) + } + + /// Every retained group was retired and unlinked by firmware without + /// executing (job list empty, stamps cover it, no recovery, fragment end + /// never written): after another VM's recovery the firmware drops an + /// innocent pair's queued groups this way (seen on 25G83 and 26A428). + /// Nothing of them reached their attachments, so the same commands are + /// published again, in order, instead of failing their owners. Only the + /// whole set is replayed; a replayed prefix would run after a live + /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS + /// apart (worker polling); the render timeout, long after publication, + /// takes it once as its retirement does. A single sample can still + /// straddle a firmware update of the fragment end record; the cost is + /// that a group which did execute runs a second time (blending twice). + fn replay_abandoned( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result { + use retirement::Memory; + if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { + return Ok(false); + } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } + use crate::g17::replay::{self, Decision, Group}; + let qids = self.qids(); + let memory = &self.memory; + let mut failure = None; + let groups = self.active.iter().map(|active| { + let innocent = + active.outer_published && !active.failed && !active.packet.completion.guilty(); + let blamed = active.packet.completion.blamed(); + let exhausted = + active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { + let mut observer = Observer { + memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof; a later poll or the timeout looks again. + let witness = |executed| { + matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) + })() + .unwrap_or_else(|error: Error| { + failure = Some(error); + (false, false, false) + }) } else { - retirement::poll(&mut front.ticket, &mut observer) + (false, false, false) }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } + Group { + innocent, + blamed, + abandoned, + exhausted, + ran, + lost, + } + }); + let decision = replay::decide(groups); + if let Some(error) = failure { + return Err(error); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + match decision { + Decision::Refuse => return Ok(false), + Decision::Wait => { + front.abandoned_since = None; + return Ok(false); + } + Decision::Replay | Decision::Fail => {} + } + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } + } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } + // decide() lets only the front be exhausted. An exhausted front fails, and its + // successors, which consume its output, fail with it rather than run without it. + let mut failing = self.active.front().is_some_and(|front| { + front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS + }); + if failing { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", + self.slot, + self.active.len() + ); + } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replaying\n", + self.slot, + self.active.len() + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + // Every group parked below shares this expiry (an early return keeps it). + self.replay_since = Some(now); + while let Some(front) = self.active.front_mut() { if !front.flist_completed { host.complete_render( (!front.qos_completed).then_some(qos), @@ -1129,74 +1432,126 @@ impl Pair { front.generation, buffer, )?; - front.qos_completed = true; front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + failing = failing || !packet.completion.begin_replay(); + if failing { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + continue; } - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - return Ok(true); + packet.reset_publication_for_replay(); + self.published_completions = None; + self.replay.push(packet).map_err(|_| EIO)?; } + Ok(true) } - /// The front group was retired and unlinked by firmware without executing - /// (job list empty, stamps cover it, no recovery, fragment end never - /// written). Nothing of this group reached its attachments, so the same - /// command is published again instead of failing the owner's VM. The - /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, - /// and only for a lone group whose completion still accepts a replay. - fn replay_abandoned( + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - use retirement::Memory; - if self.quarantined || self.active.len() != 1 || self.replay.is_some() { - return Ok(false); + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; } - let front = self.active.front().ok_or(EIO)?; - if !front.outer_published || front.failed { - return Ok(false); + Ok(true) + } + + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; let mut observer = Observer { memory: &self.memory, - qids: self.qids(), - host, + qids, + host: &mut *host, }; core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(front.ticket.ordinal)?; + let snapshot = observer.snapshot(ordinal)?; let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; - let abandoned = - retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if !abandoned { - front.abandoned_since = None; - return Ok(false); - } - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); - } - if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_info!( - "render slot {} retired without execution (pass {:?}); replaying\n", - self.slot, - snapshot.pass - ); + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } } - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; host.complete_render( (!front.qos_completed).then_some(qos), pool, @@ -1206,39 +1561,32 @@ impl Pair { front.flist_completed = true; front.qos_completed = true; } - let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - if !packet.completion.begin_replay() { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); - } - } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); - } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.replay = Some(packet); - Ok(true) + Ok(Some(true)) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). ECANCELED leaves the VM usable; any other error - /// is the ordinary device-loss outcome the replay tried to avoid. + /// publication refusal). Its successors cannot run without it and fail with + /// it. ECANCELED leaves the VM usable; any other error is the ordinary + /// device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(packet) = self.replay.take() else { - return Ok(()); - }; - defer(Deferred::Retired(packet.completion.clone(), Err(error))) + self.replay_since = None; + while let Some(packet) = self.replay.pop() { + defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; + } + Ok(()) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1357,22 +1705,51 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - if !self.quarantined && self.active.len() == 1 { - let active = self.active.front().ok_or(EIO)?; - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if active.outer_published - && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? - { + // Groups the firmware dropped unexecuted are published again, with their successors. + if self.replay_abandoned(host, defer, false)? { + return Ok(true); + } + // Fallback when a live or exhausted group prevents an ordered replay: a spared front group + // the firmware retired without executing during another VM's recovery is retired on its + // own even with successors queued, so the pair is not quarantined into a sticky EIO. It + // fails with ECANCELED, which spared work never reports as device loss; consecutive + // abandoned spared successors are retired the same way. + let front_spared = self.active.front().is_some_and(|a| { + Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() + }); + if !self.quarantined && (self.active.len() == 1 || front_spared) { + let mut retired = false; + loop { + let Some(active) = self.active.front() else { + break; + }; + let spared = active.packet.completion.spared(); + if retired && !spared { + break; + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof, as in replay_abandoned(): the group stays retained + // (quarantined with the pair if nothing was retired) instead of the error + // stopping the runtime for every client. + if !active.outer_published + || !matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + break; + } let qos = self.qos_owners(); let buffer = self.buffer_id(); + let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1384,19 +1761,30 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !active.packet.completion.spared() { + if !active.failed && !spared { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } + pr_warn!( + "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", + active.ticket.ordinal, + left + ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(EIO), + Err(if spared { ECANCELED } else { EIO }), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + retired = true; + if !spared { + break; + } + } + if retired { return Ok(true); } } @@ -1424,7 +1812,8 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - if let Some(packet) = self.replay.take() { + self.replay_since = None; + while let Some(packet) = self.replay.pop() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -1578,6 +1967,67 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; + for dependency in frontier.as_slice() { + *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -1621,6 +2071,10 @@ impl Pair { self.owner = None; } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Binding keeps the render owners and first packet explicit." + )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -1654,7 +2108,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -1670,6 +2124,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -1736,6 +2191,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index 53a0fa6e449ed7..ab96b48de3e355 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -31,6 +31,7 @@ fn include( prefix: Option, packet: &Arc, word: u64, + compute: bool, ) -> Result { if !Arc::ptr_eq(packet.completion.status(), status) { return Ok(()); @@ -38,12 +39,15 @@ fn include( if !job::Order::contains(prefix, packet.order.sequence) { return Ok(()); } - frontier.include( - prefix, - packet.order.sequence, - job::fence_status(&packet.completion.scheduler_fence()), - word, - ) + let status = job::fence_status(&packet.completion.scheduler_fence()); + frontier.include(prefix, packet.order.sequence, status, word)?; + // A compute consumer's firmware wait ends once firmware retires this group, even unexecuted + // after a restart, so it may then read output the group never wrote. Render consumers of the + // same owner follow the group on its pair, and a replay keeps that order. + if compute && status == 0 { + packet.completion.note_firmware_consumer(); + } + Ok(()) } impl Dependencies { @@ -60,12 +64,13 @@ impl Dependencies { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.word)?; + include(frontier, status, prefix, &entry.packet, entry.word, compute)?; } Ok(()) } @@ -80,7 +85,7 @@ impl Dependencies { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + include(frontier, status, prefix, &entry.packet, entry.ta_word, false)?; } Ok(()) } @@ -112,6 +117,7 @@ impl Pair { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); @@ -120,7 +126,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, compute)?; } Ok(()) } @@ -138,7 +144,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, false)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d135..756f96e3fd048d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -94,6 +97,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +106,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers)?; Ok(()) })?; } @@ -166,6 +172,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +195,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +217,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -310,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -365,6 +386,10 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Render memory construction keeps each allocation and firmware input explicit." + )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -376,10 +401,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -505,9 +531,40 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -542,21 +599,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, @@ -598,6 +644,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e077..81d4306eda28c5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,12 +25,19 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -48,11 +55,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -76,12 +92,15 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -96,11 +115,22 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Preparation forwards the render binding inputs without changing their ownership." + )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -125,7 +155,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -141,6 +171,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -190,6 +221,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, @@ -202,7 +234,9 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -213,8 +247,12 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: self.executed, + switch: None, + published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 0fc9c1d9a15dcf..c1985c72f6ec1f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,21 +73,28 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, @@ -116,7 +124,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = (&self.config as *const QueueConfig).cast::(); + let source = core::ptr::from_ref(&self.config).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index d0a2d566bdc8d3..00e4997ef3661e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,11 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -203,8 +208,12 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - let observation = observe(ticket, snapshot)?; - complete_poll(ticket, memory, snapshot, observation) + match observe(ticket, snapshot) { + Ok(observation) => complete_poll(ticket, memory, snapshot, observation), + // The borrowed snapshot can straddle a firmware cursor update like any other sample: + // take the ordinary poll's fresh snapshot and reread instead of failing the client. + Err(_) => poll(ticket, memory), + } } fn complete_poll( diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec61..da90488bfedec9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0)?; } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ba368341d84d26..3bb9d20190b57b 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,11 +9,13 @@ mod attribution; mod slot; +mod safety; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; @@ -23,20 +25,34 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -46,9 +62,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -70,10 +99,17 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; fn release_retired_compute(&mut self); + /// Ring every compute doorbell still withheld: a restart is a departure from ordinary + /// retirement, after which the firmware must see every published command. + fn announce_withheld(&mut self); } /// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions @@ -84,7 +120,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -99,6 +137,8 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, + timeouts: safety::Storm, + restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -112,8 +152,14 @@ impl State { Self { accepted: 0, pending: None, + timeouts: safety::Storm::new(), + restarts: safety::Storm::new(), } } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } + pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -185,6 +231,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -210,32 +258,78 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let cancelled = classify(host, entries, request.blamed, &mut sources); + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); + let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { - host.settle_blamed_render(qid, EIO); + pr_warn!( + "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", + sources.reason, + qid, + if blame { "failing its oldest render" } else { "attribution cleared it; spared" } + ); + if blame { + host.settle_blamed_render(qid, EIO); + } } if cancelled != 0 { host.settle_cancelled_renders(cancelled); } let scan_error = host.rescan_compute().err(); host.release_retired_compute(); + host.announce_withheld(); Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { +/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed +/// render that had not started while others had (or with no culprit at all) is innocent, and its +/// work is spared like that of every other innocent VM instead of failing with EIO. +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); @@ -245,8 +339,9 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; + let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return 0; + return (0, blame); } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -282,7 +377,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return 0; + return (0, blame); } let spared = queues.classify_work(&|status| { guilty[..count] @@ -295,5 +390,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }) .unwrap_or(Class::Spared) }); - cancelled & !(guilty_qids & !collateral) & spared + (cancelled & !(guilty_qids & !collateral) & spared, blame) } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs new file mode 100644 index 00000000000000..082416f0cb9a66 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fail closed on repeated watchdog/restart traffic, even across client loss. + +use core::sync::atomic::{AtomicU64, Ordering}; + +const WINDOW_NS: u64 = 30_000_000_000; +const LIMIT: u8 = 8; + +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + +pub(super) struct Storm { + start: Option, + count: u8, + stopped: bool, + progress: u64, +} +impl Storm { + pub(super) const fn new() -> Self { + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) + } + pub(super) fn note(&mut self, now: u64) -> bool { + if self.stopped { return true; } + if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { + self.start = Some(now); + self.count = 0; + } + self.count += 1; + self.stopped = self.count >= LIMIT; + self.stopped + } +} + +#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] +mod tests { + use super::*; + use crate::g17::{preparation::Gate, recovery::State}; + use kernel::prelude::*; + + #[test] + fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { + let gate = Gate::new().unwrap(); + let lease = gate.try_enter().unwrap(); + let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); + gate.close(); + for n in 0..64 { + let stopped = { + let mut state = state.lock(); + state.request(0, -1).unwrap(); + assert!(state.pending()); + let stopped = state.timeout_limit(n * 2_000_000_000); + if stopped { gate.fail(); } // production state -> gate order + stopped + }; + assert_eq!(stopped, n >= u64::from(LIMIT - 1)); + } + // Recovery/stop must release state before joining the retained builder. + assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), + Err(error) if error == ETIMEDOUT)); + assert!(!gate.reopen()); // failure never gives back an owned reservation + drop(lease); + gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); + assert!(!gate.reopen()); + } + #[test] + fn successful_handshakes_do_not_reset_a_restart_storm() { + let mut state = State::new(); + for n in 0..64 { + assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); + } + assert!(state.restart_limit(u64::MAX)); + } + #[test] + fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } + for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } + assert!(storm.note(64 * WINDOW_NS)); + for _ in 0..1024 { assert!(storm.note(u64::MAX)); } + } + #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] + fn replay_dependency_checks_never_wait_for_event_service() { + use crate::g17::dependency::replay_status; + assert_eq!(replay_status(0), Err(EBUSY)); + assert_eq!(replay_status(1), Ok(())); + assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 307534708e922f..bd9d4c96be14f6 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as _, + bindings::MEMREMAP_WB as usize, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs new file mode 100644 index 00000000000000..7668871376bad2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Whether a pair's retained render groups, retired by firmware without executing, are +//! published again. Only the whole set is replayed, so no replayed group runs after a live +//! successor, and work a recovery blamed is never run again. + +/// One retained group, front first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + /// Published, not failed and not classified guilty by a recovery. + pub(crate) innocent: bool, + /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): + /// it may have caused the fault. + pub(crate) blamed: bool, + /// Carries the strict abandonment witness. + pub(crate) abandoned: bool, + /// Already republished the maximum number of times. + pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Replay, + /// A group may still execute: look again later. + Wait, + /// Never replay this set; the render timeout settles it. + Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, +} + +/// Groups are read in order and the first one that decides stops the walk, so a caller can +/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its +/// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. +pub(crate) fn decide(groups: impl Iterator) -> Decision { + let mut front_exhausted = None; + let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; + for group in groups { + if !group.innocent || group.blamed { + return Decision::Refuse; + } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } + if !group.abandoned { + return Decision::Wait; + } + dropped = true; + match front_exhausted { + None => front_exhausted = Some(group.exhausted), + Some(_) => successor_exhausted |= group.exhausted, + } + } + if unordered { + return Decision::Fail; + } + match front_exhausted { + Some(true) => Decision::Replay, + Some(false) if !successor_exhausted => Decision::Replay, + _ => Decision::Refuse, + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] +mod tests { + use super::*; + fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } + } + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; + #[test] + fn every_group_abandoned_and_innocent_is_replayed() { + assert_eq!(decide([OK].into_iter()), Decision::Replay); + assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); + } + #[test] + fn nothing_retained_is_never_replayed() { + assert_eq!(decide([].into_iter()), Decision::Refuse); + } + #[test] + fn guilty_work_is_never_replayed() { + assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); + } + #[test] + fn blamed_work_is_never_replayed_even_when_spared() { + // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. + assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); + assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); + // Also once the replay limit is reached, where an innocent front alone is replayed. + let exhausted = Group { exhausted: true, ..BLAMED }; + assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); + } + #[test] + fn a_live_group_waits_and_stops_the_walk() { + assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); + // Nothing after the live group is looked at, guilty or not. + assert_eq!( + decide([group(true, false, false), group(false, true, false)].into_iter()), + Decision::Wait + ); + let mut seen = 0; + let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); + assert_eq!(decide(groups), Decision::Wait); + assert_eq!(seen, 1); + } + #[test] + fn an_exhausted_successor_refuses_an_exhausted_front_replays() { + assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); + assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); + assert_eq!( + decide([group(true, true, true), group(true, true, true)].into_iter()), + Decision::Replay + ); + } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9f..e04295d5766063 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,14 +25,38 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -44,13 +68,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -61,6 +175,10 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -68,22 +186,35 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT)] pub(super) fn new( clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { + qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, - render: render::RenderState::new(clusters, descriptor_flags)?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, + render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -96,12 +227,21 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -109,19 +249,24 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -129,23 +274,102 @@ impl Registry { Rebind::Fresh }); } + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); + } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); + } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let shared = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .share_idle()?; + if !shared { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; - for queue in self + let teardown = &self.teardown; + for mut queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -156,7 +380,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -167,13 +391,18 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { - let queue = self + let teardown = &self.teardown; + let mut queue = self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -231,7 +460,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -261,12 +490,13 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -275,12 +505,23 @@ impl Registry { Ok(freed) } + /// Builders and released queues waiting for their exact retirement or pool + /// witness temporarily occupy identities. A failed admission must let those + /// transactions finish rather than fail a healthy successor immediately. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -292,15 +533,18 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or(ENOSPC)?; + .ok_or_else(|| self.compute_admission_error( + crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -332,7 +576,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -347,13 +591,14 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; } @@ -361,6 +606,16 @@ impl Registry { } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -373,13 +628,56 @@ pub(super) enum Rebind { Fresh, } + +#[kunit_tests(asahi_g17_uninstalled_cleanup)] +mod uninstalled_cleanup_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + #[test] + fn empty_reservation_prevents_reuse_until_explicit_cancellation() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let first = registry.reserve_compute(1).unwrap(); + assert!(registry.reserved(first).is_ok()); + assert!(registry.reserve_compute(1).is_err()); + let second = registry.reserve_compute(2).unwrap(); + assert_ne!(first.id.qid(), second.id.qid()); + registry.cancel_compute(first).unwrap(); + assert!(registry.reserved(first).is_err()); + let next = registry.reserve_compute(3).unwrap(); + assert_eq!(first.id.qid(), next.id.qid()); + assert!(first.id != next.id); + } +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, + closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -389,24 +687,63 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -414,7 +751,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -422,14 +759,21 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), - }, + compute_backlog: core::sync::atomic::AtomicU32::new(0), + closed: core::sync::atomic::AtomicBool::new(false), + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), + }), GFP_KERNEL, )?, slot, @@ -438,13 +782,15 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, admission: &Admission<'_>) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -463,7 +809,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } }; let alloc = super::object::Allocator { @@ -474,7 +822,9 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -493,7 +843,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -507,10 +857,12 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, + admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -522,14 +874,18 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -547,12 +903,30 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } - fn grow_compute(&self, required: usize) -> Result { - use kernel::time::{delay::fsleep, Delta}; + /// The caller keeps its preparation lease through this two-phase rollback. + /// An empty registry entry prevents reusing the QID/free-list identity while + /// its old aliases still occupy canonical or client addresses. Recovery joins + /// this lease before removing the session, even if its epoch was invalidated. + fn finish_uninstalled_compute( + &self, retired: Option<(ComputeReservation, KBox)>, + ) -> Result { + let Some((reservation, queue)) = retired else { return Ok(()); }; + drop(queue); + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) + } + + fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -560,45 +934,83 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); - loop { + let mut growth = None; + let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } - if queue.begin_growth(&firmware.init, required)? { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + if growth.is_none() { + let notify = queue.begin_growth(&firmware.init, required)?; + // Capture physical identity before any following fallible + // operation. Close can remove the owner-keyed lookup. + growth = queue.growth_token(); + if notify { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + Err(error) if error == EAGAIN && lease.is_current() => { + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } + })(); + if result.is_err() { + if let Some(token) = growth { + let mut retired = None; + let mut state = self.shared.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + for index in 0..firmware.queues.compute.len() { + let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { + // Keep the empty entry and QID reserved until all + // graph aliases have been destroyed outside the lock. + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + break; + } + } + } + drop(state); + // Tentative replacement mappings also belong to this builder. + // Discard them before a fresh QID can acquire a successor. + drop(alias.take()); + drop(pages.take()); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + } } + result } - fn build_compute(&self, required: usize) -> Result { + fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; + self.admission_open()?; loop { - match self.shared_compute() { + match self.shared_compute(admission) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -607,10 +1019,14 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -619,13 +1035,44 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context, required)? { + self.admission_open()?; + let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { + Err(error) if error == ENOSPC => { + drop(state); + drop(lease); + self.wait_channels(observed, admission, true)?; + continue; + } + result => result?, + }; + match rebound { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; - (firmware.uat.clone(), Ok(reservation)) + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. + drop(state); + drop(lease); + self.wait_channels(observed, admission, false)?; + continue; + } + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -638,13 +1085,32 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - let ring = compute::DescriptorRing::new(&alloc)?; + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => { + drop(lease); + self.wait_channels(observed, admission, true)?; + continue; + } + result => result?, + }; let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // A competing ready owner may have taken/rearmed it while + // backing was built off-lock. Leave rejected backing local + // so it is dropped after the device guard, not in rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm()); + if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -660,10 +1126,12 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } + self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -671,8 +1139,16 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + self.compute_scheduler(&alloc, &lease, admission)?; + let queue = compute::Queue::new( + &alloc, + &objects, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + self.shared.operand_state, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -682,12 +1158,14 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - firmware.queues.install_compute(reservation, &mut built) + admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) } }; // The old aliases must disappear before a waiting constructor can @@ -703,9 +1181,11 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result != Err(EAGAIN) { - return result; + if result == Err(ENOSPC) { + self.wait_channels(observed, admission, true)?; + continue; } + if result != Err(EAGAIN) { return result; } } } } @@ -714,6 +1194,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -749,6 +1230,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -780,6 +1262,9 @@ impl DeferredBatch { drop(release); } } + pub(super) fn vacated_room(&self) -> bool { + self.vacated.len() < self.vacated.capacity() + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -799,10 +1284,18 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -814,11 +1307,33 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -839,15 +1354,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_compute_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -864,82 +1379,210 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -951,10 +1594,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -966,7 +1608,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -993,33 +1635,63 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( + registry.completion_cursor, + seen, + |record| observations.fold(record), + )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; + let mut doorbells = [0u64; 4]; + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.latch_observations(&mut registry.observations, masks)?; + let busy = !queue.channel_free(); + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + // Withheld doorbells are rung once their command is next but one; one per + // priority class announces every queue's work slots on that ring. + if let Some(doorbell) = queue.due_doorbell(|class| port.compute_announced(class)) { + let class = ((doorbell >> 2) & 3) as usize; + doorbells[class] = doorbell; + } + } + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + self.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1027,58 +1699,366 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } } impl Backend { - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { + admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + match self.build_compute(required, admission) { + Err(error) if error == EAGAIN => continue, + result => return result, + } + } + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted: the + // spared queue and its retained work stay intact past the deadline. + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } + } + + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } + + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - Err(EBUSY) + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire()) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { + return Err(render::FALLBACK); + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } + } + } + + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) } /// Called after taking this packet's publication claim. All waits and work @@ -1089,23 +2069,63 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let mut recovery_polls = 0; - let mut outer_wait = None; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. + let mut bound = true; + let mut demand = None; + let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let check = || { + if packet.cancel_requested() { return Err(ECANCELED); } + self.admission_open()?; + if packet.completion.status().get() != 0 { return Err(EIO); } + Ok(()) + }; + let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { - if packet.completion.status().get() != 0 { - return Err(EIO); + admission.check()?; + let observed = progress.observe(); + if !self.sources_published(packet)? { + self.shared.queue_events(); + self.wait_release(progress, observed, &admission)?; + continue; + } + // All DRM dependencies were satisfied before this scheduler callback. + // Recheck on every retry: pressure may yield an idle binding between + // admission and publication without changing this logical queue. + if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } + let lease = self.shared.preparations.enter_until(&admission)?; + // Work states free as this context's earlier commands retire. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + self.shared.queue_events(); + self.wait_release(progress, observed, &admission)?; + continue; + } + result => { + result?; + } } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - if packet.cancel_requested() { - return Err(ECANCELED); - } + admission.check()?; if self .shared .crashed @@ -1115,52 +2135,45 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; + drop(state); + drop(lease); + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; + if result.is_ok() { + self.remember_direct(&firmware.queues); } return result; } @@ -1173,7 +2186,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1199,6 +2212,41 @@ impl super::Firmware { } impl super::Shared { + /// Charge only real watchdog expirations, after the per-job allowance. A + /// changing client or recovery generation cannot mint a fresh storm budget. + pub(super) fn watchdog_timeout(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { return true; } + let stop = { + let mut state = self.state.lock(); + state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) + }; + if stop { self.lose_device_quietly(); } + stop + } + + /// Workload-triggered stop: halt producers first, then join only for a + /// bounded interval. A stalled borrower leaves stopped firmware and all its + /// mappings owned in state. Only a later joined stop may detach/free them. + pub(super) fn stop_runtime(&self) { + let Some(_stopping) = self.stop.try_lock() else { return; }; + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } + firmware.primary.stop(); + } + } + if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { + dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); + return; + } + self.forget_port(); + let firmware = self.state.lock().take(); + if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } + } + /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -1208,10 +2256,13 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() @@ -1245,26 +2296,63 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + let mut freed = false; + let mut releasing = false; + for mut queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); + } + if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_stalled(now)) { + return Err(ETIMEDOUT); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -1276,3 +2364,214 @@ impl super::Firmware { } } + +impl Registry { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.qids.write_capacity(f)?; + self.admission.write_capacity(f)?; + self.buffers.write_capacity(f)?; + writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; + writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; + self.render.write_capacity(f)?; + writeln!(f, "compute_slots used={} total={} reusable={} building={}", + self.compute.iter().flatten().count(), self.compute.len(), + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), + self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; + writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.write_capacity(f)?; + } + Ok(()) + } +} + +impl Backend { + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() + } + + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + admission.check()?; + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + // A short reclaim transaction obeys the same session/epoch gate as + // publication. Never keep this lease while sleeping for capacity. + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { + return Ok(()); + } + // Consumption, not publication or an elapsed timeout, frees backing. + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { + let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; + if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } + } + Ok(()) + })(); + deferred.finish(); + self.shared.queue_events(); + match result { + Err(error) if error == EAGAIN => {} + result => result?, + } + admission.check() + } +} + +#[kunit_tests(asahi_g17_shared_capacity)] +mod shared_capacity_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + + #[test] + fn all_logical_render_owners_fit_beside_shared_compute() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); + for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); + // Every accepted logical render owner now has its physical pair; no + // published identity was cancelled or retyped to achieve the fit. + } + + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init( + Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), + GFP_KERNEL, + ) + .unwrap(); + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + + #[test] + fn four_clients_have_capacity_before_any_owner_closes() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); + for client in 0..4u64 { + for queue in 0..9u64 { + let owner = 1 + client * 9 + queue; + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + } + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert_eq!(leases.len(), 36); + } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } +} + +impl Backend { + /// Logical close is independent of whether an off-lock builder has already + /// installed a physical graph. No constructor can resurrect a closed owner. + fn admission_open(&self) -> Result { + if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } + if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !self.context.is_current() { return Err(EFAULT); } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f1..d19f51c08dd290 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,13 +13,52 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } fn stop_after_error(&self) { + // Scheduler/fence callbacks never synchronously join builders or RTKit + // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - self.shared.stop_queues(); + } + + fn rollback_unpublished_compute(&self) -> Result { + // No outer preparation lease survives compute_packet's return. Hold a + // fresh epoch through alias destruction so recovery can join this work. + loop { + let lease = self.shared.preparations.enter(false)?; + let mut retired = None; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { drop(state); drop(lease); continue; } + let mut binding = None; + if let Some(index) = firmware.queues.compute.iter().position(|entry| { + entry.as_ref().and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.owner() == Some(self.owner)) + }) { + let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; + let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? + .rollback_unpublished(self.owner, &self.context)?; + binding = replaced; + if cancel { + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + } + } + drop(state); + drop(binding); + self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); + self.shared.queue_events(); + return Ok(()); + } } /// Complete all failure fanout before signalling this packet's unpublished @@ -36,6 +75,11 @@ impl Backend { } deferred.finish(); if !packet.is_published() { + if packet.engine() == Engine::Compute { + if self.rollback_unpublished_compute().is_err() { + self.stop_after_error(); + } + } packet.completion.fail_unpublished(error); } Err(error) @@ -46,6 +90,10 @@ impl Backend { } fn release_queue(&self) -> Result { + // A builder may own only an off-lock reservation when close arrives. + // Record intent before scanning queues; every later install/rearm checks it. + self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -60,12 +108,63 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -75,7 +174,11 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - Backend::ensure_compute(self, scs_pages) + // A reverse timeline chain must be submitted before the host signal. + // Validate here, but do not consume physical capacity behind unsignalled + // DRM dependencies. compute_packet admits it when the job can run. + crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + .map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -113,8 +216,13 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute and render admission wait for their shared capacity here, as a + /// fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => self.admit_render(packet), + } } fn publish(&self, packet: Arc) -> Result { @@ -142,12 +250,25 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); + if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -160,6 +281,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -177,18 +301,58 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + use crate::g17::timeout::{self, Decision}; + // As for compute, a final retirement pass precedes the attribution, and all completion + // callbacks run after releasing the device mutex. Without a collector the attribution + // goes ahead without that pass, as it did before it existed. + let mut deferred = DeferredBatch::worker().ok(); + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(false); + }; + if let Some(deferred) = deferred.as_mut() { + firmware.scan_renders(deferred)?; + } + // Borrowed-graph and queued-retirement deferrals share the renewal bound. + if timeout::transaction_grace(renewals) + && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) + { + return Ok(true); + } + let verdict = firmware.queues.render_timeout_decision(packet, renewals); + let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); + // As compute does: work still blocked behind another VM's older unfinished work + // after the whole grace is innocent, so its failure must stay non-sticky. The pair's + // quarantine extends the class to the pair's other groups. + if decision == Decision::Spare { + packet.completion.classify(crate::g17::recovery::Class::Spared); + } + if let Some(verdict) = verdict { + if let Some(blocker) = verdict.reported_blocker(renewals) { + dev_warn!(self.shared.dev.as_ref(), + "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, verdict.own.vm, verdict.own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); + } + } + Ok(decision == Decision::Defer) + })(); + if let Some(deferred) = deferred.as_mut() { + deferred.finish(); + } + matches!(result, Ok(true)) } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34aec..42282aea62023f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,26 +20,28 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; - let Some(own) = queue.timeout_head() else { - return Ok(Decision::Own); - }; - let others = firmware.queues.compute.iter().flatten() + let computes = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let decision = timeout::classify(own, renewals, others); + let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); + let verdict = timeout::attribute(own, renewals, others); + let decision = verdict.decision; if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + if let Some(blocker) = verdict.reported_blocker(renewals) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", - self.owner, own.published, renewals, decision); + "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, own.vm, own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); } Ok(decision) })(); @@ -50,7 +52,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -58,13 +63,20 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { return Ok(None); } - queue.timeout(&mut |event| deferred.push(event)) + let status = queue.timeout(&mut |event| deferred.push(event)); + // A timed-out queue leaves ordinary retirement: announce what it withheld. + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } + status })(); if result.as_ref().is_ok_and(|status| status.is_some()) { // VM status and peer settlement precede signalling this timeout. The backend @@ -85,8 +97,13 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } } Ok(()) })(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 984b6c1f643ebc..bf9d023038a51b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,6 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, + timeout::LIVENESS_NS, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, Ordering}; @@ -19,7 +20,7 @@ impl super::Registry { /// batch cannot overflow a separate cache. No allocation or wait is needed. pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { for entry in self.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { if queue.acknowledge_exit(cookie) { return; } @@ -31,12 +32,12 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.settle_exit( recovery_closed, - &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -48,7 +49,7 @@ impl Firmware { // Publication has already retained each kill. Retry only the doorbell; // failed attempts do not extend its bounded polling window. for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); } } @@ -82,17 +83,19 @@ impl Firmware { }) else { return Ok(()); }; - if self.render_control_backpressured() { + if self.render_control_backpressured() || self.kill_unsafe(context, now) { return Ok(()); } - let queue = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .ok_or(EIO)?; - let Some(owner) = queue.exit_owner(context) else { - return Ok(()); + let (owner, target) = { + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + (owner, queue.qid()) }; - let target = queue.qid(); let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { @@ -130,7 +133,7 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -147,6 +150,29 @@ impl Firmware { Ok(()) } + /// A kill while other contexts' commands run makes the firmware restart the + /// GPU, and repeated ones crash it; on an otherwise idle GPU it is safe. A + /// closed context's commands left alone retire like any others once the GPU + /// reaches them, so its queues stay quarantined and retained until then. Only + /// a GPU that retired nothing for [`LIVENESS_NS`] has stopped, and then the + /// kill proceeds. + fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); + if stalled { + return false; + } + self.queues.render.foreign_in_flight(context) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + queue.in_flight() && !queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + }) + } + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { if self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0d4a29e779b4a8..32b9acef94c0fd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -83,14 +77,14 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; + packet.check_replay_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -110,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a335..4cf2dd0fafb970 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,14 +24,17 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); @@ -128,8 +131,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -200,6 +203,22 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -217,14 +236,34 @@ impl recovery::Queues for Service<'_> { let result = self.firmware.release_compute_witnesses(self.deferred); self.settlement_error = self.settlement_error.or(result.err()); } + fn announce_withheld(&mut self) { + let mut doorbells = [0u64; 4]; + for mut queue in self + .firmware + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if let Some(doorbell) = queue.take_withheld_doorbell() { + doorbells[((doorbell >> 2) & 3) as usize] = doorbell; + } + } + // One doorbell per priority class announces every queue's work slots on that ring. + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + let result = self.firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell); + self.settlement_error = self.settlement_error.or(result.err()); + } + } } impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -232,7 +271,7 @@ impl Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -263,6 +302,10 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -272,7 +315,11 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service.firmware.port.set_generation(service.firmware.recovery.generation()); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -300,7 +347,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); + self.stop_runtime(); return; } let mut deferred = { @@ -315,8 +362,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -324,7 +373,7 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); } } @@ -344,8 +393,9 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { - self.preparations.wait_drained(); + self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -388,6 +438,11 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; + let lingers = if firmware.recovery.pending() { + None + } else { + firmware.expire_render_lingers(now)? + }; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; let pools = firmware.service_compute_pools(deferred, now)?; @@ -419,7 +474,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } @@ -430,11 +485,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } @@ -444,7 +502,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. @@ -455,6 +514,11 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf20455261..b5649f059f5d2c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; @@ -16,6 +17,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -24,6 +26,7 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -46,7 +49,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -55,6 +64,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -65,7 +80,11 @@ pub(super) struct RenderState { } impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + pub(super) fn new( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + ) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -76,6 +95,7 @@ impl RenderState { Ok(Self { config: Config { clusters, + deflake, descriptor_flags, }, entries, @@ -124,6 +144,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -157,6 +185,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -176,6 +205,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -198,7 +228,136 @@ impl RenderState { } impl Registry { - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } @@ -228,7 +387,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -262,12 +423,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -374,32 +531,26 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -419,6 +570,9 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } + fn next_publication(&mut self) -> Result { + Ok(self.firmware.port.next_publication()) + } fn publish_outer_pair( &mut self, priority: u8, @@ -458,6 +612,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary @@ -477,13 +634,20 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -516,6 +680,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -553,6 +741,44 @@ impl crate::g17::Firmware { Ok(()) } + /// A final retirement pass before a render watchdog attributes its timeout, as compute runs + /// scan_compute() first: a head that finished but whose completion the event worker has not + /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred + /// timeout or error, stay with the worker, which observes and settles them in its order. + pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()) + && entry.preparing.is_none() + && !entry.growing + && !entry.pending_timeout + && entry.pending_error.is_none() + }) + { + continue; + } + self.retire_render_slot(slot, None, deferred)?; + self.queues.render.collect_retired(slot)?; + self.settle_render_terminal(slot, deferred)?; + } + Ok(()) + } + pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -604,6 +830,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, @@ -763,9 +1005,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } @@ -816,4 +1068,103 @@ impl Registry { entry.pending_timeout = true; true } + + /// The oldest published group of every render pair, for cross-VM timeout attribution + /// (crate::g17::timeout). + pub(in crate::g17) fn render_timeout_heads( + &self, + ) -> impl Iterator + '_ { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .filter_map(|pair| pair.timeout_head()) + } + + /// A render watchdog blocked behind another VM's older unfinished work gets the same + /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU + /// stalls on one client, the others' renders then outlive the culprit's teardown instead + /// of all timing out together. + pub(in crate::g17) fn render_timeout_decision( + &self, + packet: &Arc, + renewals: u8, + ) -> Option { + let own = self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.contains_packet(packet)) + .and_then(|pair| pair.timeout_head()); + let computes = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.timeout_head()); + // Without a head of its own the timeout is its own. + own.map(|own| { + crate::g17::timeout::attribute( + own, + renewals, + Iterator::chain(self.render_timeout_heads(), computes), + ) + }) + } +} + +impl RenderState { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + self.pools.write_capacity(f)?; + for (slot, entry) in self.entries.iter().enumerate() { + if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { + context.write_capacity(f, "render", slot)?; + } + } + Ok(()) + } +} + +impl Registry { + /// Give idle private backing to another ready client under capacity pressure. + /// Keep graph/QID/context ownership; the next use rearms the same pool after + /// its consumed release, exactly as eviction-driven reclaim does. + pub(super) fn reclaim_idle_render_pools( + &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { break; } + let Some(pool) = entry.pool.as_ref() else { continue; }; + // None means this pool never acquired a firmware registration; + // otherwise require its exact final release to have been consumed. + if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) + || self.admission.owner_active(entry.reservation.owner) { continue; } + // A sibling or replacement may be building off-lock before it has + // installed its pool reference. Its logical owner also excludes us. + let mut idle = true; + for other in render.entries.iter().flatten().filter(|other| { + other.reservation.owner == entry.reservation.owner + || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) + }) { + if other.constructing || other.growing || other.preparing.is_some() + || other.pending_error.is_some() || other.pending_timeout { + idle = false; + break; + } + if let Some(pair) = other.pair.as_ref() { + if !pair.idle_for_context(pair.context())? { idle = false; break; } + } + } + if idle { + if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } + } + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 2b14b4548e0db8..e150a10e2c9bc7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -12,6 +12,7 @@ impl Registry { packet: &Arc, ) -> Result { let mut frontier = Frontier::new(); + let compute = packet.engine() == crate::g17::job::Engine::Compute; for entry in self.render.entries.iter().flatten() { if let Some(pair) = entry.pair.as_ref() { pair.collect_dependencies( @@ -19,6 +20,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { dependencies.collect( @@ -26,6 +28,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs new file mode 100644 index 00000000000000..1f42d1e6f404f9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Steady-state render publication without the device mutex. +//! +//! The device mutex lends the owner's installed pair out, together with its +//! free-list preparation and a parameter-buffer reference, and takes it back. +//! Between the two, command preparation and publication take only the port: +//! its render publication lock (the render work rings' single producer and +//! the publication order of attachment hazards), QoS accounting, the stamps +//! and the doorbell. While the pair is lent, the event worker skips it, so no +//! retirement can complete its commands or their free-list references. + +use super::super::{Backend, DeferredBatch}; +use crate::g17::{ + job::Packet, + object::Allocator, + port::{Port, RenderRoom}, + queue::render::{Pair, PublicationLease, RenderAddresses}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::{Arc, MutexGuard}, +}; + +/// A pair lent out for one publication, with what that publication holds. +pub(in crate::g17::runtime) struct Loan { + pair: Option>, + preparation: Option, + buffer: u8, + lease: PublicationLease, + addresses: RenderAddresses, + uat: Arc, +} + +/// Publication host of a lent pair. Registration, first-publication and +/// growth paths never run here: the loan requires an installed, registered +/// graph that needs no growth. +struct DirectHost<'a> { + port: &'a Port, + render: &'a MutexGuard<'a, ()>, + room: RenderRoom, + generation: u64, + activation: Option<(u16, u64)>, +} + +impl crate::g17::queue::render::Host for DirectHost<'_> { + fn next_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) + } + fn qos_publish_pair( + &mut self, + owners: [crate::g17::qos::Owner; 2], + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.port.qos_publish_pair(owners, scheduler, policy) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + Err(EAGAIN) + } + fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { + // An installed graph published its identities with its first command. + Ok(()) + } + fn publish_outer_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port + .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + // The lease leaves preparation when the pair is returned; nothing can + // observe it in between because the pair is lent. + self.activation = Some((id, generation)); + Ok(()) + } + fn publish_retained_pair( + &mut self, + priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port.expose_render_pair(self.render, &self.room, next) + } + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { + self.port + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2) + } + fn note_submission(&mut self) -> Result { + // Counted, with full effort witnessed, before the pair was lent. + Ok(()) + } + fn complete_render( + &mut self, + _owners: Option<[crate::g17::qos::Owner; 2]>, + _pool_id: u16, + _generation: u64, + _buffer_id: u8, + ) -> Result { + Err(EINVAL) + } +} + +impl crate::g17::Firmware { + /// Lends `owner`'s pair in `slot` for a publication of `packet`, or None if + /// the pair is not in the steady state the direct path requires. + fn lend_render( + &mut self, + slot: u8, + owner: u64, + packet: &Arc, + priority: u8, + generation: u64, + ) -> Result> { + if self.recovery.pending() || self.recovery.generation() != generation { + return Ok(None); + } + let Ok(entry) = self.queues.render.entry_mut(slot) else { + return Ok(None); + }; + if !entry.published + || entry.closed + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Ok(None); + } + let Some(pair) = entry.pair.as_mut() else { + return Ok(None); + }; + if !pair.matches(owner, &packet.context)? + || !pair.direct_ready(&packet.context, priority) + || pair.pool().vacant() + || !matches!(pair.host_growth_target(packet), Ok(None)) + { + return Ok(None); + } + let qids = pair.qids(); + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let (buffer, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer)?; + return if error == EAGAIN || error == EBUSY { + Ok(None) + } else { + Err(error) + }; + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let pair = match self.queues.render.take_pair(slot, owner, packet) { + Ok(pair) => pair, + Err(error) => { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer)?; + return if error == EBUSY { Ok(None) } else { Err(error) }; + } + }; + // The selected scheduler is retained through release acknowledgement, + // as on the device-mutex path. + packet.context.mark_published(); + Ok(Some(Loan { + pair: Some(pair), + preparation: Some(preparation), + buffer, + lease, + addresses, + uat: self.uat.clone(), + })) + } + + /// Takes a lent pair back and settles what its publication held: the + /// free-list preparation is activated once published, rolled back with the + /// parameter-buffer reference otherwise. + fn return_render( + &mut self, + slot: u8, + loan: &mut Loan, + activation: Option<(u16, u64)>, + packet: &Arc, + ) -> Result { + self.queues.render.return_pair(slot, &mut loan.pair)?; + if self.queues.render.take_missed_event(slot) { + self.primary.state.shared.queue_events(); + } + let entry = self.queues.render.entry_mut(slot)?; + // A failure or timeout recorded against the lent pair is settled by the + // event worker's collector, as after any other loan. + if entry.pending_error.is_some() || entry.pending_timeout { + self.primary.state.shared.queue_events(); + } + let pair = entry.pair.as_mut().ok_or(EIO)?; + if entry.closed { + pair.release_owner(); + } + if !packet.is_published() { + pair.discard_prepared(packet)?; + } + if let Some((id, generation)) = activation { + if self.queues.render.pools.activate(id, generation).is_err() { + self.primary.state.shared.lose_device_quietly(); + } + } + let preparation = loan.preparation.take().ok_or(EIO)?; + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(loan.buffer)?; + } + Ok(()) + } +} + +impl Backend { + /// Publishes `packet` into the owner's installed pair with the device + /// mutex held only to lend the pair out and take it back. None leaves the + /// packet unpublished for the device-mutex path. + pub(in crate::g17::runtime) fn render_direct( + &self, + packet: &Arc, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Option { + // A compute prefix needs the compute queue's state; leave it to the + // device-mutex path. + if packet.is_published() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || packet.order.wait_through[1].unwrap_or(0) != 0 + || self.admission_open().is_err() + { + return None; + } + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let priority = packet.context.policy().priority(); + let generation = port.generation(); + let dev = self.shared.drm().ok()?; + let mut loan = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut()?; + if !lease.is_current() { + return None; + } + match firmware.lend_render(slot, self.owner, packet, priority, generation) { + Ok(Some(loan)) => loan, + Ok(None) => return None, + Err(error) => return Some(Err(error)), + } + }; + let mut activation = None; + let result = (|| { + let pair = loan.pair.as_mut().ok_or(EIO)?; + let alloc = Allocator { + dev: &dev, + uat: &loan.uat, + }; + pair.prepare(&alloc, packet, loan.addresses)?; + let render = port.render_lock(); + if !lease.is_current() { + return Err(EBUSY); + } + let room = port.observe_render_room(&render, priority)?.ok_or(EBUSY)?; + let mut dependencies = pair.own_dependencies(self.owner, packet)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.admission.publication_barriers( + self.owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } + let mut host = DirectHost { + port, + render: &render, + room, + generation, + activation: None, + }; + let result = pair.publish( + packet.clone(), + &dependencies, + loan.lease, + &mut host, + &mut |event| deferred.push(event), + ); + activation = host.activation; + result?; + // A recovery that started meanwhile may have discarded the commands. + if port.generation() != generation || port.recovery_state()? != 0 { + return Err(EIO); + } + Ok(()) + })(); + let returned = { + let mut state = self.shared.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => { + let returned = firmware.return_render(slot, &mut loan, activation, packet); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); + } + returned + } + // Shutdown joins every preparation lease before detaching the + // session, so this cannot happen while `lease` is held. + None => Err(ENODEV), + } + }; + self.shared.note_changed(); + drop(lease); + if let Err(error) = returned { + self.shared.lose_device_quietly(); + return Some(Err(error)); + } + match result { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => { + if result.is_ok() { + self.direct_slot.store(u32::from(slot), Ordering::Relaxed); + } + Some(result) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55ca..20569af84bc7bf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + request.vm_slot == -1 || request.vm_slot as u32 == context.id() }) }) }) @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,13 +271,16 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c80018..52bdd964ca4ae7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -152,7 +154,7 @@ impl crate::g17::Shared { drop(state); drop(epoch); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); if let Some(mut pair) = pair { pair.stopped(&mut |event| { event.finish(); @@ -162,8 +164,8 @@ impl crate::g17::Shared { return Err(error); } let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); - if firmware.render_grow_pending() { + self.note_changed(); + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index f7f48e853f2252..1e3354d90dbfb9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -186,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -199,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; @@ -239,3 +265,11 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } + +impl Pools { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "render_pool_leases used={} total={} retained={}", + self.leases.iter().flatten().filter(|lease| !lease.released()).count(), + POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7ad..62ca7895a86f8e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,13 +29,14 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) @@ -44,6 +46,10 @@ impl BuildResources { } } +#[expect( + clippy::large_enum_variant, + reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." +)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -72,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -133,6 +145,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +183,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -238,10 +259,11 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; Pair::new( - &alloc, + alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -252,7 +274,9 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, + resources.config.deflake, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { @@ -277,7 +301,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -316,9 +340,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -360,7 +384,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -397,9 +421,12 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -429,9 +456,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -463,10 +490,14 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } firmware.queues.teardown.displace( pair.context(), @@ -475,7 +506,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -499,6 +530,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -513,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -556,8 +591,12 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) @@ -568,10 +607,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -583,8 +624,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -660,6 +702,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues @@ -670,7 +715,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index ce9771778acd4d..499e3858fcca18 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packet; this pass -//! prepares and publishes the same command again under the device mutex. +//! recovery, fragment end never written) and parks the packets in order; this +//! pass prepares and publishes the same commands again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,79 +27,91 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - let Some((packet, owner, qids, closed, busy)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - )) - }) - else { - continue; - }; - if busy { - continue; + // Publish the parked groups front first; a pause keeps the rest parked. + while self.replay_render_front(slot, deferred)? {} + } + Ok(()) + } + + /// Publishes the oldest parked group of `slot`; returns whether it was published. + fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { + let Some((packet, owner, qids, closed, busy, stalled)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + pair.replay_stalled(crate::g17::now_ns()), + )) + }) + else { + return Ok(false); + }; + if stalled { + return Err(ETIMEDOUT); + } + if busy { + return Ok(false); + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); } - if closed || packet.cancel_requested() || !packet.context.is_current() { + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; + return Ok(false); + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + Ok(true) } - if packet.completion.status().get() != 0 { + Err(error) if error == EBUSY || error == EAGAIN => Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, - }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - continue; - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - } - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - } + Ok(false) } } - Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b932161..24c724cb32470b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -33,7 +37,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as _, + kernel::time::msecs_to_jiffies(2000) as isize, ) }; if completed > 0 { @@ -121,6 +125,14 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { + // Unclassified work counts as innocent, and a blamed group must never be replayed. + // Work an earlier recovery spared is collateral here (classify() blames before its + // collateral check) and stays spared, so its failure stays non-sticky; the blame + // itself is recorded apart from the class, and refuses any replay of it. + packet.completion.note_blamed(); + if !packet.completion.spared() { + packet.completion.classify(crate::g17::recovery::Class::Guilty); + } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c5..1ef6742b689a78 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,13 +8,31 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; + +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -23,6 +41,280 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + sleep: bool, + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; + let started = super::super::now_ns(); + let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; + loop { + self.admission_open()?; + if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } + match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + // Atomics only: the callback runs under the pool mutex. + if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + Ok(()) + }) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + self.wait_render_capacity(&mut observed, started, sleep)?; + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + self.wait_render_capacity(&mut observed, started, sleep)?; + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire() { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire() { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + since: &mut Option, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { + if !sleep { + return Err(FALLBACK); + } + let now = super::super::now_ns(); + let deadline = since.get_or_insert(now).saturating_add(super::super::WORK_RING_WAIT_NS); + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.wait_domain(&self.shared.renders, sample, deadline, packet) + } + + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity(&self, observed: &mut Option, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = started + .saturating_add(super::super::WORK_RING_WAIT_NS) + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } + } + fn render_with_lease( &self, packet: &Arc, @@ -30,6 +322,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -37,22 +330,47 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() } + lease => lease, }; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; - let mut takeover_polls = 0; - let mut recovery_polls = 0; + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } + let mut busy_since = None; + let capacity_started = super::super::now_ns(); + let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); + let mut seen = None; + let mut states = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -65,7 +383,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -76,51 +398,71 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if state_polls == 1000 { - return Err(ETIMEDOUT); + if !sleep { + return Err(FALLBACK); + } + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + Some(sample) => { + self.wait_domain(work.returns(), sample, deadline_ns, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - match self.ensure_render(packet, slot) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { + Err(error) if error == ENOSPC => { + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; + continue; + } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; + continue; + } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); + if !sleep { + return Err(FALLBACK); } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared.preparations.wait_closure(gate, deadline_ns)?; } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -132,19 +474,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -163,6 +495,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -286,16 +628,37 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0c..e54b1f4638f587 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -37,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self @@ -120,7 +137,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01c..16bd5828265a41 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -115,8 +115,8 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .filter_map(|entry| entry.queue.as_deref()) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) @@ -190,10 +190,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c new file mode 100644 index 00000000000000..19d5b8b92f9347 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "sync_input.h" +#include "sync_scope.h" + +static int collect_leaf(struct dma_fence *leaf, + struct asahi_g17_fence_leaf *out, size_t capacity, + size_t *count) +{ + if (*count == capacity) + return -ENOSPC; + out[*count].fence = dma_fence_get(leaf); + out[*count].status = dma_fence_get_status(leaf); + ++*count; + return 0; +} + +/* Called without the device mutex: status polling may run fence callbacks. */ +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity) +{ + struct dma_fence *pending[128]; + size_t count = 0, depth = 1; + int error = 0; + + pending[0] = dma_fence_get(head); + while (depth && !error) { + struct dma_fence *node = pending[--depth]; + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + + if (asahi_g17_is_scope(node)) { + struct dma_fence *children[2]; + unsigned int n = asahi_g17_scope_children(node, children), i; + if (!n) { + /* Preserve a failed scope as a failed leaf, never drop its error. */ + if (dma_fence_get_status(node) < 0) + error = collect_leaf(node, out, capacity, &count); + } else if (depth + n > ARRAY_SIZE(pending)) { + for (i = 0; i < n; ++i) + dma_fence_put(children[i]); + error = -ENOSPC; + } else { + for (i = 0; i < n; ++i) + pending[depth++] = children[i]; + } + dma_fence_put(node); + continue; + } + + dma_fence_unwrap_for_each(leaf, &cursor, node) { + if (asahi_g17_is_scope(leaf)) { + if (depth == ARRAY_SIZE(pending)) + error = -ENOSPC; + else + pending[depth++] = dma_fence_get(leaf); + } else if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + } else { + error = collect_leaf(leaf, out, capacity, &count); + } + if (error) + break; + } + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + dma_fence_put(node); + } + while (depth) + dma_fence_put(pending[--depth]); + if (error) { + while (count) + dma_fence_put(out[--count].fence); + return error; + } + return count; +} + +void asahi_g17_put_fence(struct dma_fence *fence) +{ + dma_fence_put(fence); +} diff --git a/drivers/gpu/drm/asahi/g17/sync_input.h b/drivers/gpu/drm/asahi/g17/sync_input.h new file mode 100644 index 00000000000000..2b6e80f4b7f668 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_INPUT_H +#define ASAHI_G17_SYNC_INPUT_H + +#include + +struct dma_fence; + +struct asahi_g17_fence_leaf { + struct dma_fence *fence; + int status; +}; + +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity); +void asahi_g17_put_fence(struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_input.rs b/drivers/gpu/drm/asahi/g17/sync_input.rs new file mode 100644 index 00000000000000..1cd9a9ba761aa3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded input-fence components. Collection happens before the device mutex, +//! because polling a container's status can signal it and run callbacks. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +#[repr(C)] +pub(super) struct Leaf { + pub(super) fence: *mut bindings::dma_fence, + pub(super) status: i32, +} + +unsafe extern "C" { + fn asahi_g17_collect_fences(head: *mut bindings::dma_fence, + out: *mut Leaf, capacity: usize) -> i32; + fn asahi_g17_put_fence(fence: *mut bindings::dma_fence); +} + +impl Drop for Leaf { + fn drop(&mut self) { + // SAFETY: Successful collection transfers exactly one owned reference. + unsafe { asahi_g17_put_fence(self.fence) }; + } +} + +/// Allocation, unsupported containers, or a large prefix retain normal waits. +pub(super) fn collect(input: &Fence) -> Option> { + let mut capacity = 4; + loop { + let mut leaves = KVec::with_capacity(capacity, GFP_KERNEL).ok()?; + // SAFETY: The input owns its fence. The C helper writes at most capacity + // fully initialized records and releases every reference on failure. + let count = unsafe { + asahi_g17_collect_fences(input.raw(), leaves.as_mut_ptr(), capacity) + }; + if count == ENOSPC.to_errno() && capacity < 128 { + capacity = (capacity * 4).min(128); + continue; + } + if count < 0 { return None; } + // SAFETY: The helper returned this many initialized, owned records. + unsafe { leaves.set_len(count as usize) }; + return Some(leaves); + } +} + + +use super::fence::Submission; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{new_mutex, sync::{Arc, Mutex}}; + +pub(super) struct Source { + pub(super) submission: Arc, +} + +/// The counter is independent of a physical queue's allocation and survives +/// processor stop until the last consumer releases its binding loan. +pub(super) struct Hold(Arc); +impl Hold { + fn new(readers: &Arc) -> Result { + readers.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |count| count.checked_add(1)).map_err(|_| EOVERFLOW)?; + Ok(Self(readers.clone())) + } +} +impl Drop for Hold { + fn drop(&mut self) { self.0.fetch_sub(1, Ordering::AcqRel); } +} + +/// A new owner starts with no loans. Acquiring a loan requires exclusive access +/// and arms the hint; eligibility queries stay read-only. A validated owner +/// transition can clear the hint after all independent loans have been released. +#[repr(transparent)] +pub(super) struct ReaderHint { + possible: bool, +} +impl ReaderHint { + pub(super) const fn new() -> Self { + Self { possible: false } + } + pub(super) fn hold(&mut self, count: &Arc) -> Result { + let hold = Hold::new(count)?; + self.possible = true; + Ok(hold) + } + pub(super) fn has_readers(&self, count: &AtomicU32) -> bool { + self.possible && count.load(Ordering::Acquire) != 0 + } + pub(super) fn reset_for_owner(&mut self, count: &AtomicU32) -> Result { + if self.has_readers(count) { return Err(EBUSY); } + self.possible = false; + Ok(()) + } +} + +#[pin_data] +pub(super) struct Sources { + inputs: KVec, + publication_waits: KVec, + pub(super) sources: KVec, + #[pin] + holds: Mutex>, +} +impl Sources { + pub(super) fn new(inputs: KVec, sources: KVec, + publication_waits: KVec) -> Result> { + CONVERTED.fetch_add(1, Ordering::Relaxed); + if !publication_waits.is_empty() { PENDING.fetch_add(1, Ordering::Relaxed); } + Arc::pin_init(pin_init!(Self { + inputs, sources, publication_waits, + holds <- new_mutex!(KVec::new(), "G17 source binding loans"), + }), GFP_KERNEL) + } + pub(super) fn contains_input(&self, input: &Fence) -> bool { + self.inputs.iter().any(|fence| fence.raw() == input.raw()) + } + pub(super) fn publication_waits(&self) -> &[Fence] { &self.publication_waits } + pub(super) fn check(&self) -> Result { + for source in &self.sources { source.submission.source_status()?; } + Ok(()) + } + pub(super) fn set_holds(&self, holds: KVec) { + let old = core::mem::replace(&mut *self.holds.lock(), holds); + drop(old); + } + pub(super) fn release(&self) { self.set_holds(KVec::new()); } +} + + +static CONVERTED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PUBLISHED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PENDING: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn pending_count() -> u64 { PENDING.load(Ordering::Relaxed) } +pub(super) fn note_published() { PUBLISHED.fetch_add(1, Ordering::Relaxed); } +pub(super) fn counters() -> (u64, u64) { + (CONVERTED.load(Ordering::Relaxed), PUBLISHED.load(Ordering::Relaxed)) +} + +/// A revalidated compute completion dependency for a direct work descriptor. +#[derive(Clone, Copy)] +pub(super) struct NativeWait { + pub(super) sequence: u64, + pub(super) dependency: super::fw::kick::KickDependency, + pub(super) stamp_va: u64, + pub(super) event_slot: u32, +} +impl NativeWait { + pub(super) fn bytes(&self, consumer_stamp: u32, uuid: u64) -> [u8; 0x48] { + let mut out = [0; 0x48]; + let timestamp = self.dependency.word() & ((1u64 << 40) - 1); + for (offset, value) in [ + (0, 4u32), + (0x14, u32::from(self.dependency.qid())), + (0x18, super::fw::stamp_value(timestamp as u32)), + (0x24, self.event_slot), + (0x28, consumer_stamp), + (0x34, 1), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(4, self.stamp_va), (0xc, timestamp), (0x2c, uuid)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + out + } +} + +static NATIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn note_native(count: usize) { NATIVE.fetch_add(count as u64, Ordering::Relaxed); } +pub(super) fn native_count() -> u64 { NATIVE.load(Ordering::Relaxed) } + +#[kunit_tests(asahi_g17_input_wait)] +mod tests { + use super::*; + use crate::g17::fw::kick::{KickDependency, KickTimestamp}; + + #[test] + fn completion_record_preserves_distinct_event_and_queue_domains() { + let wait = NativeWait { + sequence: 1, + dependency: KickDependency::new(37, + KickTimestamp::new(0x12_3456_789a).unwrap()).unwrap(), + stamp_va: 0x1122_3344_5566_7788, + event_slot: 7, + }; + // Protocol fixture includes the unaligned stamp pointer, full 40-bit + // timestamp, wrapped 24-bit stamp and independent legacy event slot. + let expected: [u8; 0x48] = [ + 4, 0, 0, 0, 0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11, 0x9a, 0x78, 0x56, 0x34, + 0x12, 0, 0, 0, 37, 0, 0, 0, + 0, 0x9a, 0x78, 0x56, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 0xcc, 0xbb, 0xaa, 8, 7, 6, 5, + 4, 3, 2, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + ]; + assert_eq!(wait.bytes(0xaabb_cc00, 0x0102_0304_0506_0708), expected); + } + + #[test] + fn failed_loan_acquisition_never_wraps_or_releases_another_reader() { + let readers = Arc::new(AtomicU32::new(u32::MAX - 1), GFP_KERNEL).unwrap(); + let last = Hold::new(&readers).unwrap(); + assert!(matches!(Hold::new(&readers), Err(error) if error == EOVERFLOW)); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX); + drop(last); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX - 1); + } + + #[test] + fn loan_outlives_original_queue_counter_reference() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + drop(readers); + assert_eq!(second.0.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(second.0.load(Ordering::Acquire), 1); + drop(second); + } + + #[test] + fn reader_guard_tracks_partial_release_and_rearms_after_zero() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + assert!(!hint.has_readers(&readers)); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + assert!(matches!(hint.reset_for_owner(&readers), Err(error) if error == EBUSY)); + drop(first); + assert!(hint.has_readers(&readers)); + drop(second); + assert!(!hint.has_readers(&readers)); + hint.reset_for_owner(&readers).unwrap(); + let next = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + drop(next); + assert!(!hint.has_readers(&readers)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.c b/drivers/gpu/drm/asahi/g17/sync_output.c new file mode 100644 index 00000000000000..08b4a95f26c9b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include "sync_output.h" +#include "sync_scope.h" + +struct asahi_g17_signal_target { + struct drm_syncobj *syncobj; + struct dma_fence_chain *chain; + u64 point; +}; + +struct asahi_g17_signal_bundle { + size_t count; + size_t anchor; + bool published; + struct asahi_g17_signal_target targets[]; +}; + +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle) +{ + size_t i; + + if (!bundle) + return; + for (i = 0; i < bundle->count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + if (target->chain) + dma_fence_chain_free(target->chain); + if (target->syncobj) + drm_syncobj_put(target->syncobj); + } + kfree(bundle); +} + +/* The Rust caller supplies its validated, immutable snapshot, never a user pointer. */ +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *syncs, + size_t count, size_t anchor, int *error) +{ + struct asahi_g17_signal_bundle *bundle; + size_t i; + + *error = -EINVAL; + if (!count || anchor >= count || !syncs[anchor].point) + return NULL; + *error = -ENOMEM; + bundle = kzalloc(struct_size(bundle, targets, count), GFP_KERNEL); + if (!bundle) + return NULL; + bundle->count = count; + bundle->anchor = anchor; + for (i = 0; i < count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + target->syncobj = syncs[i].syncobj; + if (target->syncobj) + drm_syncobj_get(target->syncobj); + if (!target->syncobj) { + *error = -ENOENT; + goto fail; + } + target->point = syncs[i].point; + if (syncs[i].timeline) { + target->chain = dma_fence_chain_alloc(); + if (!target->chain) + goto fail; + } + if (i != anchor && target->syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + /* The anchor may follow the other outputs. Reject aliases in either order. */ + for (i = 0; i < count; ++i) { + if (i != anchor && bundle->targets[i].syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + *error = 0; + return bundle; +fail: + asahi_g17_signals_free(bundle); + return NULL; +} + +static void asahi_g17_signal_install(struct asahi_g17_signal_target *target, + struct dma_fence *fence) +{ + if (target->chain) { + drm_syncobj_add_point(target->syncobj, target->chain, fence, target->point); + target->chain = NULL; + } else { + drm_syncobj_replace_fence(target->syncobj, fence); + } +} + +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted) +{ + struct dma_fence *previous, *scope; + size_t i; + + if (bundle->published) + return -EINVAL; + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + scope = asahi_g17_scope_create(previous, submitted, + bundle->targets[bundle->anchor].point); + dma_fence_put(previous); + if (!scope) + return -ENOMEM; + /* Only exported completion includes earlier work. GPU execution gets no new wait. */ + asahi_g17_signal_install(&bundle->targets[bundle->anchor], scope); + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + asahi_g17_scope_bind(scope, previous); + dma_fence_put(previous); + for (i = 0; i < bundle->count; ++i) + if (i != bundle->anchor) + asahi_g17_signal_install(&bundle->targets[i], scope); + bundle->published = true; + dma_fence_put(scope); + return 0; +} + +/* Allocation or validation failed before enqueue: install an error promise without allocating. */ +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure) +{ + size_t i; + + if (bundle->published) + return; + for (i = 0; i < bundle->count; ++i) + asahi_g17_signal_install(&bundle->targets[i], failure); + bundle->published = true; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.h b/drivers/gpu/drm/asahi/g17/sync_output.h new file mode 100644 index 00000000000000..a752d9ceaf4b0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_OUTPUT_H +#define ASAHI_G17_SYNC_OUTPUT_H +#include + +struct dma_fence; +struct drm_syncobj; +struct asahi_g17_signal_bundle; +struct asahi_g17_signal_spec { + struct drm_syncobj *syncobj; + u64 point; + u32 timeline; +}; + +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *specs, + size_t count, size_t anchor, int *error); +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle); +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted); +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs new file mode 100644 index 00000000000000..928ebe23193948 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Atomic preparation of a queue completion scope and its exported signals. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +// Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these +// local permits building this module against an older kernel's Rust bindings. +pub(crate) const SYNC_CUMULATIVE: u32 = 2; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 29; + +unsafe extern "C" { + fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, + error: *mut i32) -> *mut core::ffi::c_void; + fn asahi_g17_signals_free(bundle: *mut core::ffi::c_void); + fn asahi_g17_signals_publish(bundle: *mut core::ffi::c_void, + current: *mut bindings::dma_fence) -> i32; + fn asahi_g17_signals_fail(bundle: *mut core::ffi::c_void, + failure: *mut bindings::dma_fence); +} + +#[repr(C)] +struct SignalSpec { + syncobj: *mut core::ffi::c_void, + point: u64, + timeline: u32, +} + +/// Owns syncobj references and preallocated chain nodes; never owns user pointers. +pub(crate) struct SignalBundle(*mut core::ffi::c_void); + +impl SignalBundle { + /// C retains references before the borrowed sync items can be dropped. + pub(crate) fn new(syncs: &[crate::file::SyncItem], anchor: usize) -> Result + { + let mut specs = KVec::with_capacity(syncs.len(), GFP_KERNEL)?; + for sync in syncs { + specs.push(SignalSpec { + syncobj: sync.syncobj.as_raw().cast(), point: sync.timeline_value, + timeline: u32::from(sync.timeline), + }, GFP_KERNEL)?; + } + let mut error = 0; + // SAFETY: All entries borrow live sync objects; C copies the entries and + // acquires its own references before returning. + let raw = unsafe { asahi_g17_signals_new( + specs.as_ptr(), specs.len(), anchor, &mut error) }; + if raw.is_null() { return Err(Error::from_errno(error)); } + Ok(Self(raw)) + } + + pub(crate) fn publish(&mut self, current: &Fence) -> Result { + // SAFETY: The bundle and current fence are owned for this call; C takes + // its own references on publication and reports failure before any store. + let error = unsafe { asahi_g17_signals_publish(self.0, current.raw()) }; + if error == 0 { Ok(()) } else { Err(Error::from_errno(error)) } + } + + pub(crate) fn fail(&mut self, failure: &Fence) { + // SAFETY: Live owned arguments; this only consumes uninstalled chain nodes. + unsafe { asahi_g17_signals_fail(self.0, failure.raw()) }; + } +} + +impl Drop for SignalBundle { + fn drop(&mut self) { + // SAFETY: Exactly one owner; published syncobjs retain their own fences. + unsafe { asahi_g17_signals_free(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c new file mode 100644 index 00000000000000..a3153f3122b946 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sync_scope.h" +#include "sync_input.h" + +int asahi_g17_completion_fence_status(struct dma_fence *fence) +{ + /* These private fences have no signaled callback. A pending flag is + * sufficient to keep the frontier member; a concurrent signal can only + * defer pruning until the next submission. Read terminal errors through + * the normal locked API after observing the signalled flag. + */ + if (!dma_fence_test_signaled_flag(fence)) + return 0; + return dma_fence_get_status(fence); +} + +struct asahi_g17_scope; +struct asahi_g17_scope_callback { + struct dma_fence_cb callback; + struct asahi_g17_scope *scope; +}; + +struct asahi_g17_scope { + struct dma_fence base; + atomic_t pending; + struct dma_fence *children[2]; + struct asahi_g17_scope_callback callbacks[2]; + struct irq_work work; + struct llist_node ready; + /* Identifies the exact anchor point this scope replaced. */ + u64 anchor_context; + u64 anchor_point; +}; + +static const struct dma_fence_ops scope_ops; +static DEFINE_PER_CPU(struct llist_head *, active_scopes); +#define SCOPE_ATOMIC_BATCH 64 + +bool asahi_g17_is_scope(struct dma_fence *fence) +{ + return fence && fence->ops == &scope_ops; +} + +static struct asahi_g17_scope *to_scope(struct dma_fence *fence) +{ + return container_of(fence, struct asahi_g17_scope, base); +} + +static const char *scope_driver_name(struct dma_fence *fence) +{ + return "asahi"; +} + +static const char *scope_timeline_name(struct dma_fence *fence) +{ + return "g17-queue-scope"; +} + +static bool scope_signaled(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + if (atomic_read_acquire(&scope->pending)) + return false; + cmpxchg(&fence->error, 1, 0); + return true; +} + +static bool scope_enable_signaling(struct dma_fence *fence) +{ + /* Child callbacks are already armed; no recursive callback installation. */ + return !scope_signaled(fence); +} + +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned long flags; + unsigned int i, count = 0; + + spin_lock_irqsave(&fence->inline_lock, flags); + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + if (scope->children[i]) + children[count++] = dma_fence_get(scope->children[i]); + spin_unlock_irqrestore(&fence->inline_lock, flags); + return count; +} + +static void scope_release_completed(struct asahi_g17_scope *scope) +{ + struct dma_fence *children[2]; + unsigned long flags; + unsigned int i; + + /* Readers take references under this lock. Signalled scopes need no history. */ + spin_lock_irqsave(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) { + children[i] = scope->children[i]; + scope->children[i] = NULL; + } + spin_unlock_irqrestore(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) + dma_fence_put(children[i]); + dma_fence_put(&scope->base); +} + +static void scope_complete(struct asahi_g17_scope *scope) +{ + cmpxchg(&scope->base.error, 1, 0); + dma_fence_signal(&scope->base); + scope_release_completed(scope); +} + +static void scope_work(struct irq_work *work) +{ + struct asahi_g17_scope *scope = container_of(work, struct asahi_g17_scope, work); + cmpxchg(&scope->base.error, 1, 0); + asahi_g17_signal_fence(&scope->base); + scope_release_completed(scope); +} + +static void scope_defer_list(struct llist_node *nodes) +{ + struct llist_node *node, *next; + llist_for_each_safe(node, next, nodes) { + struct asahi_g17_scope *scope = + container_of(node, struct asahi_g17_scope, ready); + /* Transfer the ready-list reference to the irq_work callback. */ + irq_work_queue(&scope->work); + } +} + +void asahi_g17_signal_fence(struct dma_fence *fence) +{ + struct llist_head ready = LLIST_HEAD_INIT(ready); + struct llist_node *nodes, *node, *next; + unsigned long flags; + unsigned int completed = 0; + + /* Fence callbacks are atomic. Keep the local ready list pinned while + * signaling, then finish ready scopes after the source fence is unlocked. + * New ready scopes append to this list instead of recursing or sending IPIs. + */ + preempt_disable(); + if (__this_cpu_read(active_scopes)) { + dma_fence_signal(fence); + preempt_enable(); + return; + } + __this_cpu_write(active_scopes, &ready); + dma_fence_signal(fence); + for (;;) { + /* An interrupt must not append after the final empty test. */ + local_irq_save(flags); + nodes = llist_del_all(&ready); + if (!nodes) + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + if (!nodes) + break; + llist_for_each_safe(node, next, nodes) { + if (completed == SCOPE_ATOMIC_BATCH) { + struct llist_node *more; + local_irq_save(flags); + more = llist_del_all(&ready); + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + preempt_enable(); + scope_defer_list(node); + scope_defer_list(more); + return; + } + scope_complete(container_of(node, struct asahi_g17_scope, ready)); + ++completed; + } + } + preempt_enable(); +} + +static void scope_arrived(struct asahi_g17_scope *scope, int error) +{ + if (error < 0) + cmpxchg(&scope->base.error, 1, error); + if (atomic_dec_and_test(&scope->pending)) { + struct llist_head *ready; + dma_fence_get(&scope->base); + preempt_disable(); + ready = __this_cpu_read(active_scopes); + if (ready) + llist_add(&scope->ready, ready); + preempt_enable(); + /* Foreign fence signalers have no driver completion pass to drain us. */ + if (!ready) + irq_work_queue(&scope->work); + } +} + +static void scope_callback(struct dma_fence *child, struct dma_fence_cb *callback) +{ + struct asahi_g17_scope_callback *cb = + container_of(callback, typeof(*cb), callback); + scope_arrived(cb->scope, child->error); + dma_fence_put(&cb->scope->base); +} + +static void scope_release(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + dma_fence_put(scope->children[i]); + dma_fence_free(fence); +} + +static void scope_set_deadline(struct dma_fence *fence, ktime_t deadline) +{ + struct asahi_g17_fence_leaf *leaves; + int count, i; + + if (dma_fence_is_signaled(fence)) + return; + /* A deadline is an advisory hint. Keep its traversal bounded and iterative, + * including when a foreign timeline boundary contains earlier scopes. + */ + leaves = kmalloc_array(128, sizeof(*leaves), GFP_ATOMIC | __GFP_NOWARN); + if (!leaves) + return; + count = asahi_g17_collect_fences(fence, leaves, 128); + for (i = 0; i < count; ++i) { + if (!asahi_g17_is_scope(leaves[i].fence)) + dma_fence_set_deadline(leaves[i].fence, deadline); + dma_fence_put(leaves[i].fence); + } + kfree(leaves); +} + +static const struct dma_fence_ops scope_ops = { + .get_driver_name = scope_driver_name, + .get_timeline_name = scope_timeline_name, + .enable_signaling = scope_enable_signaling, + .signaled = scope_signaled, + .release = scope_release, + .set_deadline = scope_set_deadline, +}; + +void asahi_g17_scope_bind(struct dma_fence *fence, struct dma_fence *root) +{ + struct asahi_g17_scope *scope = to_scope(fence); + /* A concurrent raw syncobj reset is invalid for the private queue anchor, + * but must not turn a hostile userspace race into a kernel dereference. + */ + if (!root || dma_fence_chain_contained(root) != fence) + return; + /* Written once before the submit returns; an unmatched snapshot takes fallback. */ + WRITE_ONCE(scope->anchor_point, root->seqno); + WRITE_ONCE(scope->anchor_context, root->context); +} + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point) +{ + struct dma_fence *contained = previous ? dma_fence_chain_contained(previous) : NULL; + struct dma_fence *parent = NULL; + struct asahi_g17_scope *scope, *prior = NULL; + u64 context; + unsigned int i; + + if (asahi_g17_is_scope(contained)) { + prior = to_scope(contained); + if (READ_ONCE(prior->anchor_context) != previous->context || + READ_ONCE(prior->anchor_point) != previous->seqno || + point <= contained->seqno) + prior = NULL; + } + if (prior) { + parent = dma_fence_get(contained); + context = contained->context; + } else { + /* Empty submissions or external edits may insert a foreign timeline point. + * Flatten that boundary once; the ordinary path retains just the prior scope. + */ + if (previous) { + parent = dma_fence_unwrap_merge(previous); + if (!parent) + return NULL; + } + context = dma_fence_context_alloc(1); + } + scope = kzalloc_obj(*scope); + if (!scope) { + dma_fence_put(parent); + return NULL; + } + dma_fence_init64(&scope->base, &scope_ops, NULL, context, point); + init_irq_work(&scope->work, scope_work); + scope->children[0] = parent; + scope->children[1] = dma_fence_get(submitted); + /* The constructor guard prevents completion while callbacks are being installed. */ + atomic_set(&scope->pending, 3); + scope->base.error = 1; + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) { + struct dma_fence *child = scope->children[i]; + if (!child) { + scope_arrived(scope, 0); + continue; + } + scope->callbacks[i].scope = scope; + dma_fence_get(&scope->base); + if (dma_fence_add_callback(child, &scope->callbacks[i].callback, scope_callback)) { + scope_arrived(scope, child->error); + dma_fence_put(&scope->base); + } + } + scope_arrived(scope, 0); + return &scope->base; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h new file mode 100644 index 00000000000000..e88b78aacdd7b5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_SCOPE_H +#define ASAHI_G17_SYNC_SCOPE_H +#include +struct dma_fence; +void asahi_g17_signal_fence(struct dma_fence *fence); +/* Only for explicitly signalled driver completion fences, with no poll op. */ +int asahi_g17_completion_fence_status(struct dma_fence *fence); + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point); +void asahi_g17_scope_bind(struct dma_fence *scope, struct dma_fence *root); +bool asahi_g17_is_scope(struct dma_fence *fence); +/* Returns owned child references, or zero once the scope is signalled. */ +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]); +#endif diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b1..53518a76e91a47 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index e76dab07f56db7..7b4ef4137b7958 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,11 +6,88 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// A borrowed render graph or queued off-lock retirement is not evidence of +/// progress. Share the finite deferral bound with cross-VM attribution. +pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } + +/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. +/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four +/// extra polls give it ten seconds, still one tenth of the legacy scheduler's +/// fallback. This is an execution allowance, not evidence of GPU progress, and +/// is consumed once per scheduler job, independently of cross-VM/replay grace. +pub(crate) const POLL_MS: u32 = 2000; +const EXECUTION_RENEWALS: u8 = 4; + +/// A background control/replay poll has no scheduler fence to expire it. +/// Its timestamp belongs to the exact transaction and is never renewed by work. +pub(crate) fn background_stalled(since: Option, now: u64) -> bool { + since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) +} + +/// Input comes only from fully validated commands. Render metadata covers the +/// VDM half (vertex/tessellation/geometry) and the fragment half independently. +/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. +pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { + compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build +} + +/// Firmware dependencies can hold an untagged helper behind a long command on +/// the other engine. Classify the complete validated ioctl before enqueueing +/// anything; no state or allowance leaks into a later submission. +pub(crate) fn select_submission(commands: impl Iterator) -> bool { + commands.into_iter().any(|enabled| enabled) +} + +/// Only the explicit AS-build bit is newly accepted; reserved and unrelated +/// flags remain invalid on G17P. Return None before any command is published. +pub(crate) fn as_build(flags: u32) -> Option { + const AS_BUILD: u32 = 1 << 2; + if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } +} + +pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= EXECUTION_RENEWALS { + return false; + } + *renewals += 1; + true +} + +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Renews a published job's deadline while some command retired within +/// [`LIVENESS_NS`] (`since_retired`), at most [`SHARED_RENEWALS`] times. +pub(crate) fn renew_shared(published: bool, since_retired: Option, renewals: &mut u8) -> bool { + if !published + || !since_retired.is_some_and(|since| since < LIVENESS_NS) + || *renewals >= SHARED_RENEWALS + { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - pub(crate) vm: usize, + /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. + pub(crate) vm: u64, pub(crate) published: u64, pub(crate) finished: bool, + /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains + /// it. No later watchdog of its own will settle it. + pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -20,23 +97,174 @@ pub(crate) enum Decision { Spare, } -pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - if own.finished || !others.into_iter().any(|other| { - other.vm != own.vm && !other.finished && other.published < own.published - }) { - Decision::Own - } else if renewals < MAX_RENEWALS { - Decision::Defer - } else { - Decision::Spare +/// Another VM's older unfinished work that can block `own`: live work before work whose queue +/// already failed, the oldest of each first. +pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { + others + .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) + .min_by_key(|other| (other.failed, other.published)) +} + +/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that +/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind +/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only +/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame +/// published before the culprit, which timed out first, would spare the culprit, and a pair that +/// never drains would spare every later timeout. +pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { + match blocker { + _ if own.finished => Decision::Own, + None => Decision::Own, + Some(_) if renewals < MAX_RENEWALS => Decision::Defer, + Some(blocker) if blocker.failed => Decision::Own, + Some(_) => Decision::Spare, + } +} + +/// A watchdog's decision with the heads it rests on, for its log line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Verdict { + pub(crate) decision: Decision, + pub(crate) own: Head, + pub(crate) blocker: Option, +} + +impl Verdict { + /// The blocker a watchdog line names: at the first deferral and once the grace has run out. + pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { + self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) + } +} + +pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { + let blocker = blocker(own, others); + Verdict { + decision: decide(own, renewals, blocker), + own, + blocker, } } -#[cfg(test)] +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; - fn head(vm: usize, published: u64, finished: bool) -> Head { - Head { vm, published, finished } + #[test] + fn selects_compute_both_render_halves_and_builds() { + assert!(!select(false, [false; 2], false)); + assert!(select(true, [false; 2], false)); + assert!(select(false, [true, false], false)); + assert!(select(false, [false, true], false)); + assert!(select(false, [true; 2], false)); + assert!(select(false, [false; 2], true)); + } + #[test] + fn helpers_share_only_the_current_submission_allowance() { + assert!(select_submission([false, true, false].into_iter())); + assert!(select_submission([true, false].into_iter())); + assert!(select_submission([false, true].into_iter())); + assert!(!select_submission([false, false].into_iter())); + assert!(!select_submission([].into_iter())); + let mut helper = 0; + for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } + assert!(!renew_execution(true, true, &mut helper)); + assert!(!renew_execution(true, false, &mut 0)); + } + #[test] + fn build_hint_rejects_reserved_and_unknown_flags() { + assert_eq!(as_build(0), Some(false)); + assert_eq!(as_build(1 << 2), Some(true)); + for bit in 0..32 { + if bit == 2 { continue; } + assert_eq!(as_build(1 << bit), None); + assert_eq!(as_build((1 << bit) | (1 << 2)), None); + } + assert_eq!(as_build(u32::MAX), None); + } + #[test] + fn borrowed_render_and_queued_retirement_cannot_defer_forever() { + for count in 0..=u8::MAX { + assert_eq!(transaction_grace(count), count < MAX_RENEWALS); + } + } + #[test] + fn background_control_and_replay_polls_have_an_absolute_deadline() { + let since = 42; + assert!(!background_stalled(None, u64::MAX)); + assert!(!background_stalled(Some(since), since - 1)); + assert!(!background_stalled(Some(since), since + 9_999_999_999)); + for _ in 0..1024 { + assert!(background_stalled(Some(since), since + 10_000_000_000)); + } + assert!(background_stalled(Some(since), u64::MAX)); + } + fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + attribute(own, renewals, others).decision + } + fn head(vm: u64, published: u64, finished: bool) -> Head { + Head { vm, published, finished, failed: false } + } + fn failed(vm: u64, published: u64) -> Head { + Head { vm, published, finished: false, failed: true } + } + #[test] + fn ray_query_execution_is_bounded_at_ten_seconds() { + let mut renewals = 0; + for poll in 1..=4 { + assert!(renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, poll); + } + for _ in 0..256 { + assert!(!renew_execution(true, true, &mut renewals)); + } + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); + } + #[test] + fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { + for (published, enabled) in [(false, false), (false, true), (true, false)] { + let mut renewals = 0; + assert!(!renew_execution(published, enabled, &mut renewals)); + assert_eq!(renewals, 0); + } + } + #[test] + fn ray_query_and_cross_vm_credits_are_independent_and_finite() { + let mut ray = 0; + let own = head(5, 101, false); + for _ in 0..4 { + assert!(renew_execution(true, true, &mut ray)); + } + for victim in 0..MAX_RENEWALS { + assert!(!renew_execution(true, true, &mut ray)); + assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); + } + #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, Some(4_900_000_000), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, Some(0), &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, Some(LIVENESS_NS), &mut renewals)); + assert!(!renew_shared(true, None, &mut renewals)); + assert!(!renew_shared(false, Some(0), &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] + fn exhausted_or_invalid_credit_never_wraps() { + for value in 4..=u8::MAX { + let mut renewals = value; + assert!(!renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, value); + } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -52,6 +280,53 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] + fn a_blocker_that_already_failed_defers_then_never_spares() { + // An innocent frame published before the culprit timed out first and failed; its + // quarantined pair keeps its head until the firmware drains it. + let culprit = head(7, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); + } + assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); + assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); + // A pair that never drains does not spare any later timeout either. + let drained_never = [failed(5, 100), failed(5, 102)]; + for own in [head(7, 105, false), head(9, 200, false)] { + assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); + } + } + #[test] + fn a_live_blocker_still_spares_beside_a_failed_one() { + let own = head(9, 103, false); + let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; + assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); + assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); + // Failed or not, a blocker must be older, unfinished and of another VM. + let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; + assert_eq!(blocker(own, late.into_iter()), None); + assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); + // Among failed blockers the oldest is reported. + let both_failed = [failed(8, 102), failed(5, 100)]; + assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); + } + #[test] + fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { + let own = head(9, 103, false); + let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); + let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; + assert_eq!(verdict, expected); + assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); + assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); + let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); + assert_eq!(last.decision, Decision::Own); + assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); + let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); + assert_eq!(spared.decision, Decision::Spare); + assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); + // Nothing blocks: nothing to report. + assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); + } + #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c new file mode 100644 index 00000000000000..23666be389b969 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "trace_api.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +bool asahi_g17_trace_publish_enabled(void) +{ + return trace_asahi_g17_publish_enabled(); +} + +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence) +{ + trace_asahi_g17_publish(engine, qid, ordinal, ctx, seq, + fence ? fence->context : 0, fence ? fence->seqno : 0, + __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end) +{ + if (trace_asahi_g17_complete_enabled()) + trace_asahi_g17_complete(engine, qid, ordinal, ctx, seq, gpu_start, + gpu_end, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) +{ + trace_asahi_g17_bind(engine, qid, ctx, kind); +} + +bool asahi_g17_trace_doorbell_enabled(void) +{ + return trace_asahi_g17_doorbell_enabled(); +} + +void asahi_g17_trace_doorbell(u8 engine, u32 power) +{ + trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_message(u8 kind, u64 value) +{ + if (trace_asahi_g17_message_enabled()) + trace_asahi_g17_message(kind, value, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_event(u8 phase) +{ + if (trace_asahi_g17_event_enabled()) + trace_asahi_g17_event(phase, __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h new file mode 100644 index 00000000000000..7e5a50e774b598 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Command lifecycle tracepoints of the G17 runtime. Times are joined by + * (ctx, seq); `counter` is the CPU's virtual counter and `gpu_start`/`gpu_end` + * come from the firmware's completion record, so a reader can place firmware + * dispatch and completion on the host timeline. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM asahi_g17 + +#if !defined(ASAHI_G17_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define ASAHI_G17_TRACE_H + +#include + +TRACE_EVENT(asahi_g17_publish, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 fence_context, u64 fence_seqno, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, fence_context, fence_seqno, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->fence_context = fence_context; + __entry->fence_seqno = fence_seqno; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu fence=%llu:%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->fence_context, __entry->fence_seqno, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_complete, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, gpu_start, gpu_end, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, gpu_start) + __field(u64, gpu_end) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->gpu_start = gpu_start; + __entry->gpu_end = gpu_end; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu gpu=%llu..%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->gpu_start, __entry->gpu_end, __entry->counter) +); + +TRACE_EVENT(asahi_g17_bind, + TP_PROTO(u8 engine, u8 qid, u32 ctx, u8 kind), + TP_ARGS(engine, qid, ctx, kind), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ctx) + __field(u8, kind) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ctx = ctx; + __entry->kind = kind; + ), + TP_printk("engine=%u qid=%u ctx=%u kind=%u", + __entry->engine, __entry->qid, __entry->ctx, __entry->kind) +); + +TRACE_EVENT(asahi_g17_doorbell, + TP_PROTO(u8 engine, u32 power, u64 counter), + TP_ARGS(engine, power, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u32, power) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->power = power; + __entry->counter = counter; + ), + TP_printk("engine=%u power=%#x counter=%llu", __entry->engine, __entry->power, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_message, + TP_PROTO(u8 kind, u64 value, u64 counter), + TP_ARGS(kind, value, counter), + TP_STRUCT__entry( + __field(u8, kind) + __field(u64, value) + __field(u64, counter) + ), + TP_fast_assign( + __entry->kind = kind; + __entry->value = value; + __entry->counter = counter; + ), + TP_printk("kind=%u value=%#llx counter=%llu", __entry->kind, __entry->value, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_event, + TP_PROTO(u8 phase, u64 counter), + TP_ARGS(phase, counter), + TP_STRUCT__entry( + __field(u8, phase) + __field(u64, counter) + ), + TP_fast_assign( + __entry->phase = phase; + __entry->counter = counter; + ), + TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) +); + +#endif + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#include diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs new file mode 100644 index 00000000000000..b76b46425d2ce9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a +//! static-key test in C while its event is disabled. + +use kernel::{bindings, dma_fence::RawDmaFence, sync::Arc}; + +unsafe extern "C" { + fn asahi_g17_trace_publish_enabled() -> bool; + fn asahi_g17_trace_publish(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + fence: *mut bindings::dma_fence); + fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + gpu_start: u64, gpu_end: u64); + fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_message(kind: u8, value: u64); + fn asahi_g17_trace_doorbell_enabled() -> bool; + fn asahi_g17_trace_doorbell(engine: u8, power: u32); + fn asahi_g17_trace_event(phase: u8); +} + +pub(super) const RENDER: u8 = 0; +pub(super) const COMPUTE: u8 = 1; + +/// Called once the command's doorbell has been rung. +pub(super) fn publish(engine: u8, qid: u8, ordinal: u32, packet: &Arc) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_publish_enabled() } { + return; + } + let fence = packet.completion.fence(); + // SAFETY: `fence` holds a reference for the duration of the call. + unsafe { + asahi_g17_trace_publish(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + fence.raw()) + }; +} + +/// Called when exact completion with GPU timestamps is retired. +pub(super) fn complete(engine: u8, qid: u8, ordinal: u32, packet: &super::job::Packet, + gpu: [u64; 2]) +{ + // SAFETY: Plain values only. + unsafe { + asahi_g17_trace_complete(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + gpu[0], gpu[1]) + }; +} + +/// A physical queue was bound to a logical owner: 0 fresh, 1 rebound, 2 same VM. +pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; +} + +/// A work doorbell is about to ring; `power` samples the core power state +/// only while the event is enabled. +pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_doorbell_enabled() } { + return; + } + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_doorbell(engine, power()) }; +} + +pub(super) const MAILBOX: u8 = 0; +pub(super) const CONTROL: u8 = 1; +pub(super) const EFFORT: u8 = 2; + +/// A host-to-firmware message: a mailbox message, a device-control record by +/// opcode, or a new performance effort value. +pub(super) fn message(kind: u8, value: u64) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_message(kind, value) }; +} + +/// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. +pub(super) fn event(phase: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_event(phase) }; +} diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h new file mode 100644 index 00000000000000..d09f529a163f0f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Entry points of the G17 command lifecycle tracepoints, called from Rust + * (g17/trace.rs). + */ +#ifndef _ASAHI_G17_TRACE_API_H +#define _ASAHI_G17_TRACE_API_H + +#include + +struct dma_fence; + +bool asahi_g17_trace_publish_enabled(void); +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence); +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end); +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind); +bool asahi_g17_trace_doorbell_enabled(void); +void asahi_g17_trace_doorbell(u8 engine, u32 power); +void asahi_g17_trace_message(u8 kind, u64 value); +void asahi_g17_trace_event(u8 phase); + +#endif /* _ASAHI_G17_TRACE_API_H */ diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef5..b0f2861382ad74 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -90,10 +146,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +164,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } @@ -143,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 3bcf8a9692dcca..72238159d04752 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,8 +21,10 @@ use kernel::{ }, // }, error::Result, + new_mutex, prelude::*, sync::aref::ARef, + sync::Mutex, uapi, // }; @@ -40,6 +42,7 @@ use crate::{ AsahiDrmDriver, // }, file, + memacct, mmu, }; @@ -60,6 +63,11 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, + /// Memory accounting and process attribution of the held backing. + charge: memacct::Charge, + /// Serializes overlapping timestamp writes through every binding of this BO. + #[pin] + pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -74,6 +82,9 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. +/// +/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object +/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -99,6 +110,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } + gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -106,7 +118,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result<&'a kernel::scatterlist::SGTable> { +) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -115,9 +127,7 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(gem) - { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { return Err(EINVAL); } Ok(()) @@ -211,6 +221,7 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -229,20 +240,23 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -fn checked_object_size(size: usize) -> Result { - if size == 0 { return Err(EINVAL); } +pub(crate) fn checked_object_size(size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } - fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { + let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - checked_object_size(size)?, + size, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, + pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -250,6 +264,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -259,12 +274,17 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. +/// +/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the +/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, + pages_no_reclaim: bool, + commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -276,6 +296,7 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, + pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -283,6 +304,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + charge: memacct::Charge::user_current(commit), }, )?; @@ -308,16 +330,22 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), + charge: args.charge, + timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. + /// + /// Their memory is not counted: an import's backing belongs to, and is charged by, its + /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, + charge: memacct::Charge::none(), }) } @@ -372,6 +400,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether CPU mappings use the cacheable WRITEBACK policy. + pub(crate) fn is_writeback(&self) -> bool { + self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 + } + pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } @@ -439,3 +472,22 @@ extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } #[cfg(not(CONFIG_DRM_ASAHI_G15))] pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b14..be65af433a6332 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -268,6 +267,11 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; @@ -285,6 +289,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + *crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -303,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { if self.queue_limits().is_some() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + if self.soft_faults() && self.supports_soft_faults() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags @@ -312,32 +320,39 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. + fn vm_destroyed(&self) {} /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 /// T8122 runtime admits any client. @@ -350,6 +365,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -359,12 +381,12 @@ pub(crate) trait Gpu: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -752,6 +774,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -1427,15 +1450,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0bd..2650131a4a2936 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 00000000000000..7a4d6f288dc3da --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), + // Measured on the 20-core M4 Pro; other core counts are refused at probe. + core_layout: CoreLayout { + cores: Some((2, 20)), + operand_state: fw::compute::OperandState::TWO_BY_TEN, + deflake: fw::render::Deflake::TWO_CLUSTERS, + }, +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f22..75925713bf10bb 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,10 +4,11 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -23,10 +24,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,13 +36,30 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + // The layouts the A18 Pro has always run with. + core_layout: crate::g17::CoreLayout { + cores: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -67,6 +85,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } @@ -273,7 +293,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -384,6 +404,9 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; + /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the + /// policy word of the user classes is kept (no macOS trace of class 1 yet). + pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -393,7 +416,9 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + // One physical pair per logical owner leaves room for every execution + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 8fc7791e939ba1..dff1af0783965d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -5,11 +5,11 @@ //! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for //! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 //! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client -//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at -//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks -//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore -//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a -//! VM, so a refused client can still query the parameters but never reaches the hardware. +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields +//! of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks GET_PARAMS +//! for at least the structure up to them; older userspace asks for less. On G15 a file therefore +//! has to ask for those fields before it can create a VM. Every path to the GPU needs a VM, so a +//! refused client can still query the parameters but never reaches the hardware. use core::sync::atomic::{ AtomicU32, @@ -29,15 +29,20 @@ use crate::{ mmu, }; -/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. -const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; +/// GET_PARAMS size an AGX3-aware client asks for: the global parameter structure up to the end +/// of `gpu_hal_generation`. The optional tail fields that follow it are not part of the check, +/// so growing the structure does not refuse userspace built against the shorter one. +const AGX3_PARAMS_SIZE: u64 = + (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + + core::mem::size_of::()) as u64; +const _: () = assert!(AGX3_PARAMS_SIZE == 600); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; static REFUSALS: AtomicU32 = AtomicU32::new(0); -/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the -/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for at least +/// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { { diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809c..bcb9fd32c7d255 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -254,7 +254,8 @@ impl gpu::Gpu for Backend { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } - fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + crate::timestamp::Buffer::firmware( + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range)?) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ffc57c77afcb20..1212113af1618a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -18,12 +18,13 @@ impl Destination { let guard = objects.lock(); let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; match &*object { - file::Object::TimestampBuffer(mapping) => { + file::Object::TimestampBuffer(buffer) => { + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { return Err(EINVAL); } - Ok(Some(Self { mapping: mapping.clone(), offset })) + Ok(Some(Self { mapping, offset })) } } } diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 7a5587ef6ec786..5f0ab2b4aea4f5 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,39 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +/// +/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per +/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second +/// UAT with another policy would change it for both. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,12 +108,16 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -167,6 +201,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -185,7 +223,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = ((asid as u64) << 48) + let val = (u64::from(asid) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs new file mode 100644 index 00000000000000..ac26439d8a3f96 --- /dev/null +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU memory attribution, accounting and admission limits. +//! +//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the +//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of +//! the task that faulted it in, but the OOM killer scores a task by the resident set +//! of its mm, and GPU-only pages are mapped into no process. A game holding +//! gigabytes of buffers therefore scores like a tiny process, and so does a +//! compositor; which one an OOM kill hits is left to the other terms of the score. +//! +//! This module closes that gap: +//! +//! - Every user object remembers the mm of the process that created it (an `mmgrab` +//! reference, which keeps only the `mm_struct` itself). While the GPU holds the +//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. +//! They are shmem pages, so this is where `oom_badness()`, the OOM report's +//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages +//! are uncharged when idle reclaim or purge releases the backing and when the +//! object is freed; the mm reference is dropped after the last uncharge, so the +//! counter is back to its own value before `__mmdrop()` checks it. +//! - Each client (DRM file) and the device keep totals of created and held bytes, +//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the +//! `gpu_memory` sysfs file next to `agx_stats`. +//! - A new user object is admitted only while its client stays within a per-client +//! limit and the device within a device-wide limit, both fractions of RAM. +//! A refused allocation fails with `ENOMEM`, which userspace reports as +//! out-of-device-memory, instead of pushing the machine into a global OOM. +//! +//! Imported dma-bufs are charged to their exporter and are not counted here. +//! Kernel objects (firmware structures, per-queue buffers and the global heaps) +//! count towards the device total and its limit, but are attributed to no process. +//! +//! The limits and the RSS attribution apply only to G17 for now: only its probe +//! calls [`Ledger::configure`]. Other GPUs keep the totals, with no limit and no +//! attribution. + +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + mm::{ + Mm, + RssCounter, // + }, + page::PAGE_SIZE, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::module_parameters; + +/// Device-wide GPU memory totals, in bytes. +/// +/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). +#[repr(C)] +pub(crate) struct Ledger { + /// Size of every live user object (created, not imported). + pub(crate) user_committed: AtomicU64, + /// Size of the live user objects whose backing the GPU currently holds. + pub(crate) user_resident: AtomicU64, + /// Size of the live kernel objects whose backing the GPU holds. + pub(crate) kernel_resident: AtomicU64, + /// User object creations refused by a limit. + pub(crate) refused: AtomicU64, + /// Device-wide limit on user committed plus kernel resident bytes (0: none). + pub(crate) limit: AtomicU64, + /// Per-client limit on committed bytes (0: none). + pub(crate) client_limit: AtomicU64, +} + +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(Ledger, refused) == 24); + assert!(core::mem::offset_of!(Ledger, client_limit) == 40); + assert!(core::mem::size_of::() == 48); +}; + +/// The ledger of the (single) AGX GPU. +pub(crate) static LEDGER: Ledger = Ledger::new(); + +/// Whether held user objects are charged to their creator's RSS. Set by [`Ledger::configure`]. +static ATTRIBUTE_RSS: AtomicBool = AtomicBool::new(false); + +/// The limits that follow from the amount of RAM and two percentages. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct Limits { + /// Device-wide limit in bytes (0: none). + pub(crate) device: u64, + /// Per-client limit in bytes (0: none). + pub(crate) client: u64, +} + +fn pct_of(total: u64, pct: u32) -> u64 { + match pct { + 0 => 0, + // Above 100% cannot be held anyway; treat it as "no limit". + p if p >= 100 => 0, + p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, + } +} + +/// Computes the limits for `total_ram` bytes of RAM. +/// +/// A percentage of 0 or 100 and above disables that limit. A per-client limit +/// is never larger than the device limit. +pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { + let device = pct_of(total_ram, device_pct); + let mut client = pct_of(total_ram, client_pct); + if device != 0 && (client == 0 || client > device) { + client = device; + } + Limits { device, client } +} + +/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). +/// +/// Returns whether the bytes were added. Concurrent callers cannot overshoot. +fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { + let mut cur = counter.load(Ordering::Relaxed); + loop { + let Some(new) = cur.checked_add(bytes) else { + return false; + }; + if limit != 0 && new.saturating_add(extra) > limit { + return false; + } + match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => return true, + Err(actual) => cur = actual, + } + } +} + +fn sub(counter: &AtomicU64, bytes: u64) { + let prev = counter.fetch_sub(bytes, Ordering::AcqRel); + if prev < bytes { + pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + counter.store(0, Ordering::Relaxed); + } +} + +impl Ledger { + pub(crate) const fn new() -> Self { + Self { + user_committed: AtomicU64::new(0), + user_resident: AtomicU64::new(0), + kernel_resident: AtomicU64::new(0), + refused: AtomicU64::new(0), + limit: AtomicU64::new(0), + client_limit: AtomicU64::new(0), + } + } + + /// Sets the limits and the RSS attribution from the module parameters and the amount of RAM. + pub(crate) fn configure(&self) { + // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. + let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; + // SAFETY: `si_meminfo()` only fills in the structure it is given. + unsafe { kernel::bindings::si_meminfo(&mut info) }; + let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); + let limits = limits_for( + total, + *module_parameters::mem_limit_pct.value(), + *module_parameters::client_mem_limit_pct.value(), + ); + self.set_limits(limits); + ATTRIBUTE_RSS.store( + *module_parameters::oom_attribution.value() != 0, + Ordering::Relaxed, + ); + pr_info!( + "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + limits.device >> 20, + limits.client >> 20 + ); + } + + /// Admission capacities exported in the optional GET_PARAMS tail. + pub(crate) fn limits(&self) -> Limits { + Limits { + device: self.limit.load(Ordering::Relaxed), + client: self.client_limit.load(Ordering::Relaxed), + } + } + + pub(crate) fn set_limits(&self, limits: Limits) { + self.limit.store(limits.device, Ordering::Relaxed); + self.client_limit.store(limits.client, Ordering::Relaxed); + } + + /// Admits a new user object of `bytes` for `client`. + /// + /// Reserves the bytes against both limits; the returned [`Commit`] gives + /// them back when it is dropped. + pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { + let client_limit = self.client_limit.load(Ordering::Relaxed); + if !try_add_within(&client.committed, bytes, client_limit, 0) { + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + // Kernel objects share the device limit but are never refused. + let kernel = self.kernel_resident.load(Ordering::Relaxed); + if !try_add_within( + &self.user_committed, + bytes, + self.limit.load(Ordering::Relaxed), + kernel, + ) { + sub(&client.committed, bytes); + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + Ok(Commit { + ledger: self, + client: client.clone(), + bytes, + }) + } +} + +/// Per-client (DRM file) totals, in bytes. +pub(crate) struct Client { + /// Size of the live user objects this client created. + committed: AtomicU64, + /// Size of those whose backing the GPU currently holds. + resident: AtomicU64, +} + +impl Client { + pub(crate) fn new() -> Result> { + Ok(Arc::new( + Self { + committed: AtomicU64::new(0), + resident: AtomicU64::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn committed(&self) -> u64 { + self.committed.load(Ordering::Relaxed) + } + + pub(crate) fn resident(&self) -> u64 { + self.resident.load(Ordering::Relaxed) + } +} + +/// Bytes of a user object reserved against the limits. +pub(crate) struct Commit { + ledger: &'static Ledger, + client: Arc, + bytes: u64, +} + +impl Drop for Commit { + fn drop(&mut self) { + sub(&self.client.committed, self.bytes); + sub(&self.ledger.user_committed, self.bytes); + } +} + +enum Kind { + /// An object the DRM core created (a dma-buf import): charged to its exporter. + None, + /// A driver-owned object. + Kernel(&'static Ledger), + /// A user object, attributed to the process that created it. + User { + commit: Commit, + mm: Option>, + }, +} + +/// The accounting state of one GEM object. +pub(crate) struct Charge { + kind: Kind, + bytes: u64, + /// Whether the bytes are currently counted as held by the GPU. + held: AtomicBool, +} + +// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. +unsafe impl Send for Charge {} +// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. +unsafe impl Sync for Charge {} + +impl Charge { + /// An object that is not counted (a dma-buf import). + pub(crate) fn none() -> Self { + Self { + kind: Kind::None, + bytes: 0, + held: AtomicBool::new(false), + } + } + + /// A kernel object of `bytes`. + pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { + Self { + kind: Kind::Kernel(ledger), + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object admitted by `commit`, attributed to `mm` if there is one. + pub(crate) fn user(commit: Commit, mm: Option>) -> Self { + let bytes = commit.bytes; + Self { + kind: Kind::User { commit, mm }, + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object attributed to the current process, if the ledger attributes RSS. + pub(crate) fn user_current(commit: Commit) -> Self { + let mm = if ATTRIBUTE_RSS.load(Ordering::Relaxed) { + kernel::current!().mm().map(|mm| ARef::from(&**mm)) + } else { + None + }; + Self::user(commit, mm) + } + + fn pages(&self) -> isize { + isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) + } + + fn apply(&self, held: bool) { + match &self.kind { + Kind::None => {} + Kind::Kernel(ledger) => { + if held { + ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&ledger.kernel_resident, self.bytes); + } + } + Kind::User { commit, mm } => { + if held { + commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); + commit + .ledger + .user_resident + .fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&commit.client.resident, self.bytes); + sub(&commit.ledger.user_resident, self.bytes); + } + if let Some(mm) = mm { + let pages = self.pages(); + mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); + } + } + } + } + + /// Records that the GPU now holds (or no longer holds) the object's backing. + /// + /// Idempotent: only a change of state is counted. + pub(crate) fn set_held(&self, held: bool) { + if self.held.swap(held, Ordering::AcqRel) != held { + self.apply(held); + } + } +} + +impl Drop for Charge { + fn drop(&mut self) { + // Uncharge before `kind` (and with it the mm reference) is dropped. + self.set_held(false); + } +} + +#[kunit_tests(asahi_memacct)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + const MIB: u64 = 1 << 20; + + #[test] + fn default_limits_follow_ram() { + // The module parameter defaults. + let l = limits_for(8 * GIB, 67, 50); + assert_eq!(l.device, 8 * GIB * 67 / 100); + assert_eq!(l.client, 4 * GIB); + } + + #[test] + fn disabled_and_clamped_limits() { + assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); + assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); + // A client limit above the device limit is clamped to it. + let l = limits_for(8 * GIB, 25, 50); + assert_eq!(l.client, l.device); + // No client limit with a device limit means the device limit. + let l = limits_for(8 * GIB, 50, 0); + assert_eq!(l.client, 4 * GIB); + // No device limit leaves the client limit alone. + let l = limits_for(8 * GIB, 0, 25); + assert_eq!(l, Limits { device: 0, client: 2 * GIB }); + } + + #[test] + fn percentages_do_not_overflow() { + let l = limits_for(u64::MAX, 99, 98); + assert!(l.device > l.client); + assert!(l.device <= u64::MAX / 100 * 99 + 99); + } + + #[test] + fn try_add_respects_limit_and_extra() { + let c = AtomicU64::new(0); + assert!(try_add_within(&c, 6, 10, 0)); + assert!(!try_add_within(&c, 5, 10, 0)); + assert!(try_add_within(&c, 4, 10, 0)); + assert_eq!(c.load(Ordering::Relaxed), 10); + let d = AtomicU64::new(0); + assert!(!try_add_within(&d, 6, 10, 5)); + assert!(try_add_within(&d, 5, 10, 5)); + let e = AtomicU64::new(u64::MAX - 1); + assert!(!try_add_within(&e, 2, 0, 0)); + assert!(try_add_within(&e, 1, 0, 0)); + } + + static ADMIT: Ledger = Ledger::new(); + + #[test] + fn admission_reserves_and_releases() -> Result { + ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let c = Client::new()?; + let a1 = ADMIT.admit(&a, 40 * MIB)?; + // The client limit refuses a's second 40 MiB; nothing stays reserved. + assert!(ADMIT.admit(&a, 40 * MIB).is_err()); + assert_eq!(a.committed(), 40 * MIB); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); + let b1 = ADMIT.admit(&b, 50 * MIB)?; + // The device limit refuses c's 20 MiB and rolls back c's reservation. + assert!(ADMIT.admit(&c, 20 * MIB).is_err()); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); + assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); + drop(a1); + assert_eq!(a.committed(), 0); + let c1 = ADMIT.admit(&c, 20 * MIB)?; + drop((b1, c1)); + assert_eq!(b.committed(), 0); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + static KERNEL: Ledger = Ledger::new(); + + #[test] + fn kernel_objects_count_against_the_device_limit() -> Result { + KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); + let k = Charge::kernel(&KERNEL, 48 * MIB); + k.set_held(true); + let a = Client::new()?; + assert!(KERNEL.admit(&a, 32 * MIB).is_err()); + let c = KERNEL.admit(&a, 16 * MIB)?; + drop(k); + assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); + drop(c); + Ok(()) + } + + static HELD: Ledger = Ledger::new(); + + #[test] + fn held_transitions_are_idempotent() -> Result { + HELD.set_limits(Limits { device: 0, client: 0 }); + let a = Client::new()?; + let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); + assert!(!charge.held.load(Ordering::Relaxed)); + charge.set_held(true); + charge.set_held(true); + assert_eq!(a.resident(), 8 * MIB); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); + charge.set_held(false); + charge.set_held(false); + assert_eq!(a.resident(), 0); + charge.set_held(true); + // Dropping a held object uncharges it and returns its commit. + drop(charge); + assert_eq!(a.resident(), 0); + assert_eq!(a.committed(), 0); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); + assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + #[test] + fn imports_are_not_counted() { + let charge = Charge::none(); + charge.set_held(true); + assert!(charge.held.load(Ordering::Relaxed)); + drop(charge); + } + + static MM: Ledger = Ledger::new(); + + // `mm_alloc()` is only exported to modules in the KUnit namespace. + #[cfg(not(MODULE))] + #[test] + fn held_pages_are_charged_to_the_creating_mm() -> Result { + MM.set_limits(Limits { device: 0, client: 0 }); + // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. + let raw = unsafe { kernel::bindings::mm_alloc() }; + if raw.is_null() { + return Err(ENOMEM); + } + // SAFETY: `raw` is a live mm until the `mmput()` below. + let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); + let a = Client::new()?; + let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); + assert_eq!(mm.rss_counter(RssCounter::Anon), 0); + charge.set_held(false); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + // Freeing a held object uncharges it before dropping its mm reference. + drop(charge); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + drop(mm); + // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then + // checks that every RSS counter is back to zero. + unsafe { kernel::bindings::mmput(raw) }; + Ok(()) + } +} + +#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] +mod exported_limits_tests { + use super::*; + + #[test] + fn exported_capacities_match_admission() { + let ledger = Ledger::new(); + assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); + let limits = limits_for(8 << 30, 67, 50); + ledger.set_limits(limits); + assert_eq!(ledger.limits(), limits); + assert!(limits.client < limits.device); + assert_eq!(limits.client, 4 << 30); + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 3a7941b4a36534..9e06244b83ac6a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -115,14 +114,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +156,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -377,6 +435,8 @@ struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } @@ -389,7 +449,20 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -508,22 +581,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -570,6 +637,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -655,7 +723,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -698,7 +767,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -733,14 +807,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -750,6 +817,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -804,15 +884,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -840,10 +924,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -851,7 +960,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -909,8 +1025,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -988,7 +1103,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -996,8 +1111,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1038,6 +1154,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1078,10 +1200,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1124,29 +1243,87 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); + +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { + use kernel::drm::gem::BaseObject; + self.0.owner().size() + } + + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + return; + } + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.0.owner().timestamp_lock.lock(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let base = self.0.addr.cast::(); + let pointer = unsafe { base.add(offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: All eight bytes were checked above; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.0.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); + } +} + +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, + mapping: TimestampVmap, offset: usize, } +#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - pub(crate) fn write(&self, value: u64) { - let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1170,14 +1347,24 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)? + .page_table + .translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1188,10 +1375,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1306,10 +1495,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1399,6 +1588,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1441,8 +1643,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1451,6 +1672,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1467,6 +1690,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1475,14 +1703,44 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1496,11 +1754,15 @@ impl UatShared { } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1517,20 +1779,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1583,9 +1963,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1668,16 +2048,36 @@ impl HandoffFlush { fn prefetch_sgt_segments( dev: &device::Device, gem: &gem::Object, -) -> Result> { +) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { segments, _lease: sgt.into_lease() }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1700,12 +2100,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1731,6 +2131,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1785,7 +2186,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1798,13 +2202,16 @@ impl Vm { /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1922,9 +2329,7 @@ impl Vm { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } @@ -1985,6 +2390,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1999,17 +2435,16 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let size = object_range.range(); let mut vm_exec = unsafe { @@ -2017,14 +2452,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2045,7 +2472,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2063,9 +2490,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2083,17 +2517,16 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(&gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2105,14 +2538,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2147,9 +2572,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2167,17 +2599,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2195,14 +2626,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2245,6 +2668,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2288,22 +2727,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2508,8 +2960,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2522,9 +2984,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2555,6 +3018,18 @@ pub(crate) struct AdoptedRegions { } impl Uat { + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. fn map_physical(range: &Range, min_size: usize) -> Result { @@ -2604,10 +3079,24 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, + ) -> Result { + Self::new_adopted_with_user_upper(dev, geometry, regions, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, ) -> Result { let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; @@ -2619,8 +3108,12 @@ impl Uat { let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2629,10 +3122,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2644,8 +3141,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2748,7 +3251,10 @@ impl Uat { } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2786,7 +3292,9 @@ impl Uat { } fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); @@ -2809,16 +3317,24 @@ impl Uat { let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2827,8 +3343,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2847,6 +3367,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for index in UAT_USER_CTX_START..UAT_NUM_CTX { + if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2898,6 +3447,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2907,14 +3457,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2936,7 +3489,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -2956,7 +3509,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -2972,15 +3525,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2997,24 +3554,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3024,7 +3590,12 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -3033,12 +3604,16 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3047,10 +3622,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3066,12 +3645,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3079,11 +3673,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3134,11 +3728,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3150,7 +3748,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3159,8 +3757,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3178,3 +3780,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = lock_vm(&self.gpuvm, None, false)?; + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 44a1575ecfd2b7..e222c7cd8e6a98 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,6 +17,7 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, + _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -57,7 +58,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state without taking DMA reservations or changing any leaf. + /// Prepares host state before taking the GPUVM exec lock or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -80,24 +81,20 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - crate::gem::held_sg_table(gem, dev)?; + let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - { - let mut segments = vm_bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetch_sgt_segments(dev, gem)?); - } - } + prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, + _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 30010be497973d..66d507c2d76949 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,9 +79,13 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 && state.commits == 0 - && !state.blocked && !state.quarantined && !state.draining - && !state.closed && !state.cleanup_failed + state.active == 0 + && state.commits == 0 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed }) } @@ -339,10 +343,12 @@ impl Vm { drop(previous); } - /// Whether userspace has permanently closed this VM's map admission. - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed + /// admission for good). G15 releases its per-VM state after that. pub(crate) fn is_closed(&self) -> bool { - self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -411,7 +417,10 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); + return Ok(VmJobGuard { + vm: self.clone(), + _residency: residency, + }); } lifetime.changed.wait(&mut state); } @@ -471,7 +480,11 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { + pub(super) fn defer_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + start: usize, + ) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -485,7 +498,8 @@ impl Vm { } if batch .ops - .iter().skip(start) + .iter() + .skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -496,16 +510,21 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = + self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + if !state + .unmaps + .iter() + .any(|old| old.iova == unmap.iova && old.size == unmap.size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -530,16 +549,30 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { return Err(ENOENT); } - if !state.blocked { return Ok(false); } - if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -634,8 +667,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); - let objects = core::mem::replace(&mut state.objects, KVec::new()); + let unmaps = core::mem::take(&mut state.unmaps); + let objects = core::mem::take(&mut state.objects); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -671,7 +704,9 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { status.record(error); } + if let Some(status) = &self.status { + status.record(error); + } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a8134268..8611221af492a3 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs new file mode 100644 index 00000000000000..7cdad5e1f3831d --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/ppl.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire +//! guard lifetime: firmware exclusion alone does not serialize two AP threads. +//! This core has no kernel dependencies so the production guard can be exercised +//! against deterministic firmware and concurrent host transports. + +pub(super) trait Memory { + fn firmware_interested(&self) -> bool; + fn firmware_turn(&self) -> bool; + fn ap_interest(&self, interested: bool); + fn give_firmware_turn(&self); + fn barrier(&self); + fn expired(&self) -> bool; + fn relax(&self); + /// Latch failure before the transport releases host exclusion on timeout. + fn fail(&self); +} + +/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. +#[must_use = "retain the guard until every protected table mutation has completed"] +pub(super) struct Guard { + memory: T, +} + +impl Guard { + pub(super) fn acquire(memory: T) -> Result { + let timeout = || { + memory.ap_interest(false); + memory.barrier(); + memory.fail(); + Err(()) + }; + memory.ap_interest(true); + memory.barrier(); + while memory.firmware_interested() { + if memory.expired() { + return timeout(); + } + if memory.firmware_turn() { + memory.ap_interest(false); + memory.barrier(); + while memory.firmware_turn() { + if memory.expired() { + return timeout(); + } + memory.relax(); + } + memory.ap_interest(true); + memory.barrier(); + } + memory.relax(); + } + memory.barrier(); + Ok(Self { memory }) + } + + pub(super) fn memory_mut(&mut self) -> &mut T { + &mut self.memory + } +} + +impl Drop for Guard { + fn drop(&mut self) { + self.memory.give_firmware_turn(); + self.memory.barrier(); + self.memory.ap_interest(false); + self.memory.barrier(); + // Only after these stores complete may `memory` release the host mutex. + } +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 5370b87ccf1b5d..49c2241bc399fa 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,6 +251,8 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} +// SAFETY: Shared callbacks use the VM's residency/reservation locks. The +// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -273,7 +275,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -281,23 +283,36 @@ impl VmShrinker { Ok(owner) } + /// Count reclaimable pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker. The core must + /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() as _ + owner.vm.idle_reclaim_count() } + /// Reclaim idle pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker, and `sc` must + /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - let target = unsafe { (*sc).nr_to_scan } as usize; - owner.vm.reclaim_idle_pages(target) as _ + // SAFETY: The shrinker core supplies a live initialized control. + let target = unsafe { (*sc).nr_to_scan }; + owner.vm.reclaim_idle_pages(target) } } @@ -348,7 +363,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -361,11 +376,17 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Purgeable objects count on busy VMs too; idle backing only on idle ones. - if object.idle_reclaim_candidate() - && (object.madv_locked() > 0 || idle) - && object.sole_sg_lease_releasable_locked() - { + // Count retained backing on idle VMs, and purgeable contents even if idle + // reclaim already released the table. Never wait on the BO mutex under resv. + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + let advised = object.madv_locked() > 0; + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + (advised || object.has_sg_table_locked()) + && object.unleased_pages_releasable_locked() + }; + if object.idle_reclaim_candidate() && (advised || idle) && releasable { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -385,7 +406,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -396,7 +417,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -408,9 +429,12 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_some() && !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -467,7 +491,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) || !self .lifetime .as_ref() @@ -482,7 +506,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -494,9 +518,14 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_none() || !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + // A competing lease may have made the previous release fail after we + // dropped our segments. Retry that table, but do not count empty backing. + object.has_sg_table_locked() && object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index b9e2975c898ff4..fec7a6b61ad8fb 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: gem as *const gem::Object as usize, + object: core::ptr::from_ref(gem) as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = gem as *const gem::Object as usize; + let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 38a2268137effb..a66a82d92a5830 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -409,7 +409,7 @@ impl> GpuObject { }; let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index b88353a2f73441..c5a47c24ec9286 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -53,19 +54,20 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). +/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local +/// CTR_EL0 read can select too large a stride if the task migrates during the loop. #[inline(always)] fn dcache_line_size() -> usize { - let ctr: u64; - // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. - unsafe { - core::arch::asm!( - "mrs {ctr}, ctr_el0", - ctr = out(reg) ctr, - options(nomem, nostack, preserves_flags) - ) - }; - 4 << ((ctr >> 16) & 0xf) + use kernel::bindings; + let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) + | (bindings::SYS_CTR_EL0_Op1 << 16) + | (bindings::SYS_CTR_EL0_CRn << 12) + | (bindings::SYS_CTR_EL0_CRm << 8) + | (bindings::SYS_CTR_EL0_Op2 << 5); + // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine + // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. + let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; + 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -95,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -116,20 +117,114 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). +/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -137,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -253,6 +351,8 @@ impl Prot { Self { memattr, ..*self } } + /// The protection bits of a leaf with this protection (no address, table type or non-global + /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() @@ -278,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -296,6 +412,12 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Ownership could not be relinquished to firmware; physical tables remain until reboot. + retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -305,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -317,45 +438,49 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + retained: false, + reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb osh`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// `dsb sy`. Call it before anything is mapped and before any context names the root. pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -366,6 +491,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -381,7 +507,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -392,15 +520,23 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + retained: false, + reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } + /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim + /// this tree merely because a userspace handle closed or a fence reported an error. + pub(crate) fn retain(&mut self) { + self.retained = true; + } + pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -470,6 +606,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -513,55 +650,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); } - })?; + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -597,13 +737,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -611,7 +748,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -636,16 +778,9 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { + fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } + tables.contains(&phys) } else { true }; @@ -653,32 +788,44 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; + let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; tables.swap_remove(index); } + if self.cleans_poc() { + // Complete the parent descriptor clear before its detached child can be reused. + // Callers have already removed this tree from live contexts before freeing tables. + poc_barrier(); + } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -769,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -788,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -814,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -861,6 +1019,9 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { + if self.retained { + return; + } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index a68352828cfbc3..20396f2e772252 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.clone(), + mapping: mapping.firmware_mapping()?, offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 0872a50ce5d463..4abdd3e485f726 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -262,7 +262,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { QueueTimeout { dev: self.dev.clone(), id: self.id } @@ -754,7 +755,22 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - cmdbuf.skip(header.size as usize); + let payload_size = header.size as usize; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; + if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; + if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + _ => cmdbuf.skip(payload_size), + } nr_commands += 1; match header.cmd_type as u32 { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index fb22569ea5fdff..6a7bd5057fa543 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,10 +13,11 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, c_str, + bindings, + c_str, device::Core, - devres::Devres, error::{from_err_ptr, to_result}, + devres::Devres, io::{ mem::IoMem, Io, @@ -222,7 +223,10 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + from_err_ptr(bindings::apple_mbox_get( + pdev.as_ref().as_raw(), + index.try_into()?, + ))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -235,7 +239,10 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { dev: pdev.into(), mbox }) + Ok(Self { + dev: pdev.into(), + mbox, + }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -251,7 +258,11 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + dev_err!( + self.dev.as_ref(), + "Could not stop GPU coprocessor: {:?}\n", + error + ); } } } @@ -560,7 +571,10 @@ impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + let sgx = Arc::new( + sgx_req.iomap_sized::()?.into_devres()?, + GFP_KERNEL, + )?; Ok(G17Resources { dev: pdev.into(), @@ -693,6 +707,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { @@ -701,7 +720,6 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } - } /// Split the packed per-cluster core masks into one mask per cluster. @@ -732,11 +750,7 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); + dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); return Err(EIO); } @@ -744,7 +758,10 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision( + dev: &platform::Device, + id_version: u32, +) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -758,3 +775,29 @@ fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevis } }) } + +/// Device-managed SGX mapping retained by the G16 firmware session. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[derive(Clone)] +pub(crate) struct G16Resources { + sgx: Arc>>, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl G16Resources { + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; + Ok(Self { + sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, + }) + } + pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_read32(offset) + } + #[allow(dead_code)] + pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_write32(value, offset) + } +} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 9f0baa4350d93f..7d7d785a831334 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -32,9 +32,21 @@ struct asahi_stats_snapshot { u64 jobs; }; +/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ +struct asahi_gpu_memory { + u64 user_committed; + u64 user_resident; + u64 kernel_resident; + u64 refused; + u64 limit; + u64 client_limit; +}; + struct asahi_stats_attribute { struct device_attribute attr; + struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; + const struct asahi_gpu_memory *memory; bool export_enabled; }; @@ -72,20 +84,52 @@ static ssize_t agx_stats_show(struct device *dev, return n; } +/* + * GPU memory held by the driver, in bytes: user objects created (committed) + * and held by the GPU (resident), held kernel objects, the limits that admit + * user objects (0: none) and the number of refused creations. + */ +static ssize_t gpu_memory_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, memory_attr); + const struct asahi_gpu_memory *mem = stats->memory; + ssize_t n = 0; + + n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", + (unsigned long long)READ_ONCE(mem->user_committed)); + n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", + (unsigned long long)READ_ONCE(mem->user_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", + (unsigned long long)READ_ONCE(mem->kernel_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", + (unsigned long long)READ_ONCE(mem->limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", + (unsigned long long)READ_ONCE(mem->client_limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", + (unsigned long long)READ_ONCE(mem->refused)); + + return n; +} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle) + int export_enabled, const void *memory, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || (export_enabled && !snapshot)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); + BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -102,17 +146,70 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->snapshot = snapshot; stats->export_enabled = export_enabled; + sysfs_attr_init(&stats->memory_attr.attr); + stats->memory_attr.attr.name = "gpu_memory"; + stats->memory_attr.attr.mode = 0444; + stats->memory_attr.show = gpu_memory_show; + stats->memory = memory; + ret = device_create_file(dev, &stats->attr); if (ret) { kfree(stats); return ret; } + ret = device_create_file(dev, &stats->memory_attr); + if (ret) { + device_remove_file(dev, &stats->attr); + kfree(stats); + return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); +/* + * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores + * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores + * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs + * about 1 ms, so userspace may lengthen it on AC power. + */ +static u32 *asahi_idle_word; + +static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); +} + +static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + u32 ms; + + if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) + return -EINVAL; + WRITE_ONCE(*asahi_idle_word, ms); + return count; +} + +static DEVICE_ATTR_RW(idle_off_delay_ms); + +int asahi_sysfs_register_idle(struct device *dev, u32 *word) +{ + int ret; + + if (!dev || !word) + return -ENODEV; + asahi_idle_word = word; + ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); + if (ret) + asahi_idle_word = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); + void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -121,7 +218,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) return; /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); + if (asahi_idle_word) { + device_remove_file(dev, &dev_attr_idle_off_delay_ms); + asahi_idle_word = NULL; + } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 5969a1718925f1..f1873f9be6caaa 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -6,7 +6,8 @@ struct device; int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle); + int export_enabled, const void *memory, void **handle); +int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 079623d1ef2108..cee5d52a4e2619 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,7 +3,7 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::stats::StatsSnapshot; +use crate::{memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; extern "C" { @@ -11,12 +11,30 @@ extern "C" { dev: *mut c_void, snapshot: *const StatsSnapshot, export_enabled: c_int, + memory: *const Ledger, handle: *mut *mut c_void, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } +extern "C" { + fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; +} + +/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). +pub(crate) fn register_idle( + dev: *mut kernel::bindings::device, + word: *mut u32, +) -> kernel::error::Result { + // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes + // it with READ_ONCE/WRITE_ONCE. + let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; + if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } +} + /// Register only after initialization succeeds, keeping the snapshot Arc alive. +/// +/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, @@ -30,6 +48,7 @@ pub(crate) fn register( dev.cast(), snapshot, c_int::from(export_enabled), + &crate::memacct::LEDGER, &mut handle, ) }; diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs new file mode 100644 index 00000000000000..9f13436b4caf55 --- /dev/null +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared timestamp destinations shared by native command completion paths. + +use crate::mmu; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use crate::{file, gem}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::ops::Range; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::sync::atomic::{fence as barrier, Ordering}; +use kernel::{prelude::*, sync::Arc}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use kernel::{uapi, xarray}; + +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + +/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. +/// CPU-retired timestamps retain their backing without consuming global firmware address space. +pub(crate) enum Buffer { + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Cpu { + // The GEM pin keeps the object's held SG table alive through CPU publication. + gem: gem::KernelMappingPin, + range: Range, + }, +} + +impl Buffer { + pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) + } + + /// Retain a CPU-written byte range without publishing any firmware address. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { + let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { + return Err(EINVAL); + } + let gem = gem::KernelMappingPin::new(&bo.gem); + let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let _ = gem::held_sg_table(&gem, parent.as_ref())?; + Ok(Self::Cpu { gem, range }) + } + + pub(crate) fn size(&self) -> usize { + match self { + Self::Firmware(buffer) => buffer.mapping.size(), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { range, .. } => range.end - range.start, + } + } + + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { gem, range, .. } => { + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) + } + } + } + + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. + pub(crate) fn firmware_mapping(&self) -> Result> { + match self { + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { + let offset = usize::try_from(offset).map_err(|_| EINVAL)?; + let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; + if aligned && offset % size_of::() != 0 { + return Err(EINVAL); + } + if end > size { + return Err(if aligned { EINVAL } else { ERANGE }); + } + Ok(offset) +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +struct Destination { + // Retain the bound range and backing along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + aligned: bool, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { + _mapping: mapping, + cpu, + })) + } + + fn write(&self, ticks: u64, frequency_hz: u64) { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / frequency_hz) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); + self.cpu.write(ns); + barrier(Ordering::Release); + } +} + +/// Timestamp mappings resolved and retained before any command is enqueued. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct Destinations(Option; 4]>>); + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destinations { + /// G16 retains its eight-byte destination alignment and EINVAL contract. + pub(crate) fn resolve_aligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, true) + } + + fn resolve_inner( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + aligned: bool, + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + /// Publish measured timestamps before signalling completion or releasing dependent work. + pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { + for (destination, ticks) in self.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks, frequency_hz); + } + } + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = self + .0 + .as_deref() + .map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573cc..a5f9455b23f5fd 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -217,3 +217,17 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } + +/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +#[inline(always)] +pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__pinned_init(slot) } +} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e50..9de2e7f0f4f97c 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,8 +87,16 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + /* A bulk pin must not run direct reclaim once per folio when + * the driver can instead return an allocation failure. Clearing + * DIRECT_RECLAIM also makes memcg charges fail without reclaim + * or invoking the OOM killer. Preserve placement and kswapd. + */ + if (shmem->pages_no_reclaim) + gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); + mapping_set_gfp_mask(obj->filp->f_mapping, gfp); } return 0; @@ -612,10 +620,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + return !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == 1 && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1075,6 +1083,10 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; + /* Purged contents cannot be restored by rebuilding an empty table. */ + if (shmem->madv < 0) + return ERR_PTR(-EINVAL); + drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa68..30791d328550de 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,6 +37,13 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; + /** + * @pages_no_reclaim: Disable direct reclaim during backing population. + * Set before drm_gem_shmem_init(). Drivers that can fail a large pin + * request cheaply may opt in; background reclaim remains enabled. + */ + bool pages_no_reclaim; + /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -145,7 +152,9 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index d311b92620b864..66d51881993838 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,6 +212,20 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; + + /** + * @memory_limit: Device-wide limit on user committed plus kernel + * resident bytes. Zero means no driver limit. This is a capacity, + * not current free memory; other clients share it. + */ + __u64 memory_limit; + + /** + * @client_memory_limit: Per-DRM-file limit on user committed bytes. + * Zero means no driver limit. Userspace must zero-initialize the reply + * buffer: older kernels leave these optional tail fields untouched. + */ + __u64 client_memory_limit; }; /** @@ -348,6 +362,19 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, + + /** + * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept + * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. + */ + DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -806,7 +833,8 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Reserved for future extension. + * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails + * with -EACCES. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -880,6 +908,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** @@ -1614,6 +1652,17 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + + /** + * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, + * including compute-based transfers. Requires BOUNDED_AS_BUILD. + * This is an execution hint, not proof of progress or permission to + * access additional memory. The kernel validates the command normally + * and caps its execution allowance at ten seconds (four extra 2s polls). + * Other commands in the same submission receive the same allowance so + * ordered helpers cannot time out while waiting for the build. + */ + DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 099e64b3ce5fdb..9ab7494251e289 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index b5540997bd20d4..d9933c4595682a 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,6 +23,18 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } +__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, + long value) +{ + add_mm_counter(mm, member, value); +} + +__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, + int member) +{ + return get_mm_counter_sum(mm, member); +} + __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f2450492..e63b62b54a17d4 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: None, + show_fdinfo: Some(drm::File::::show_fdinfo_callback), fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8a56f00203f568..88709a7f9b36dd 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,6 +12,23 @@ use crate::{ }; use core::marker::PhantomData; +/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. +/// +/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct MemoryStats { + /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). + pub shared: u64, + /// Memory private to the client (part of `drm-total-*`). + pub private: u64, + /// Memory currently resident (`drm-resident-*`). + pub resident: Option, + /// Resident memory userspace marked purgeable (`drm-purgeable-*`). + pub purgeable: Option, + /// Memory in use by active submissions (`drm-active-*`). + pub active: Option, +} + /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -22,6 +39,11 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; + + /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. + fn memory_stats(&self) -> Option { + None + } } /// An open DRM File. @@ -48,6 +70,12 @@ impl File { self.0.get() } + /// Whether this file holds the device's DRM master (the compositor or display server). + pub fn is_current_master(&self) -> bool { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. + unsafe { bindings::drm_is_current_master(self.as_raw()) } + } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -99,6 +127,47 @@ impl File { 0 } + /// The `show_fdinfo` callback of a `struct drm_driver`. + pub(crate) extern "C" fn show_fdinfo_callback( + printer: *mut bindings::drm_printer, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose + // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the + // file, so `postclose_callback()` cannot run concurrently. + let file = unsafe { File::::from_raw(raw_file) }; + let Some(stats) = file.inner().memory_stats() else { + return; + }; + + let mut supported = 0; + let mut raw = bindings::drm_memory_stats { + shared: stats.shared, + private: stats.private, + resident: 0, + purgeable: 0, + active: 0, + }; + if let Some(resident) = stats.resident { + raw.resident = resident; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; + } + if let Some(purgeable) = stats.purgeable { + raw.purgeable = purgeable; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; + } + if let Some(active) = stats.active { + raw.active = active; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; + } + + // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` + // and the region name outlive the call. + unsafe { + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + }; + } + /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483e..83eca852139beb 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,6 +82,12 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, + /// Fail backing population when it would need direct reclaim. + /// + /// Background reclaim and ordinary allocation placement are preserved. + /// Intended for large, fallible pin requests, not essential kernel backing. + pub pages_no_reclaim: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -94,13 +100,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, + pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -116,12 +122,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// Devres object for unmapping any SGTable on driver-unbind. - sgt_res: ManuallyDrop>>>, + /// One cleanup registration for the C SG-table cache; never holds a table borrow. + sgt_cleanup: ManuallyDrop>>>, #[pin] - /// Lock for protecting initialization of `sgt_res`. + /// Serializes cleanup registration and table acquisition. sgt_lock: Mutex<()>, - /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, #[pin] inner: T, @@ -189,13 +195,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_res` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + // `sgt_cleanup` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -273,67 +279,59 @@ impl Object { self.make_vmap() } - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA - /// pages for this object. + /// Acquire a revocable lease on this object's DMA scatter-gather table. /// - /// This will pin the object in memory. It is expected that `dev` should be a pointer to the - /// same [`device::Device`] which `self` belongs to, otherwise this function will return - /// `Err(EINVAL)`. - pub fn sg_table<'a>( - &'a self, - dev: &'a device::Device, - ) -> Result<&'a scatterlist::SGTable> { + /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. + /// The table borrow cannot outlive the lease. The device must be this object's parent. + /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. + pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // SAFETY: `import_attach` is set before an imported object is published and does not - // change until the object is freed. - if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // The SG table of an imported object is the dma-buf attachment's mapping, which the - // exporter made for this device at import time. It belongs to the attachment, and - // `drm_gem_shmem_release()` hands it back to the exporter through - // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` - // would unmap and free it as if these helpers had built it. - // - // SAFETY: `sgt` of an imported object is set before the object is published and stays - // valid and unmodified until the object is freed, which `'a` cannot outlive. + // Release and purge set the reclaim bit only while they hold this object's reservation. + // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. + while let Err(e) = self.acquire_sgt_access() { + if e != EBUSY { + return Err(e); + } + drop(DmaResvGuard::new(self)); + } + // Every failure after incrementing the count drops this lease. + let lease = SGLease { obj: self.into() }; + + // SAFETY: Import attachment ownership is immutable after publication. + let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // Imports belong to the exporter. Never register them for Rust devres cleanup, + // rebuild their tables, or release their backing. + // SAFETY: The attachment table stays valid until this object is freed. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - // SAFETY: See above. - return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); - } - - // The borrowed reference can outlive any lock, so pin the backing permanently. - self.acquire_sgt_access(true)?; - - let sgt_res = 'out: { - // Fast path: sgt_res is already initialized - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; - } - - // Slow path: Grab the lock and see if we need to initialize sgt_res. + sgt + } else { + // Register cleanup once, but cache the actual table only in C's shmem->sgt. + // Reclaim clears that pointer. No Rust reference to it survives without a lease. let _guard = self.sgt_lock.lock(); - - // If someone initialized it while we were waiting, we can exit early. - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; + if self.sgt_cleanup.as_ref().is_none() { + self.sgt_cleanup.populate(Devres::new( + dev, + Ok::<_, Error>(SGTableCleanup { obj: self.into() }), + )?); } - - // If not, finish initializing and return. `populate()` cannot return false, as - // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. - self.sgt_res - .populate(Devres::new(dev, SGTableMap::new(self))?); - - // SAFETY: We just populated sgt_res above. - unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; + // SAFETY: The live object and lease exclude backing release. The helper locks + // the reservation and either returns the cached table or rebuilds it. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? }; - Ok(sgt_res.access(dev)?) + Ok(SGTable { + // SAFETY: The lease excludes release; the Bound device excludes devres revocation. + table: unsafe { scatterlist::SGTable::from_raw(sgt) }, + lease, + }) } /// Create a new shmem-backed DRM object of the given size. @@ -349,7 +347,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -360,6 +358,10 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + // SAFETY: The object is initialized but not published. The C initializer + // reads this policy when setting up the backing mapping's GFP mask. + unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; + // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -403,7 +405,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -429,23 +431,18 @@ impl Object { } } - fn acquire_sgt_access(&self, permanent: bool) -> Result { + fn acquire_sgt_access(&self) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - let next = if permanent { - state | SGT_PERMANENT - } else { - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } - state + 1 - }; + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } match self.sgt_access.compare_exchange_weak( state, - next, + state + 1, Ordering::AcqRel, Ordering::Acquire, ) { @@ -466,11 +463,34 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } + /// Whether the object still has a cached device SG table. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn has_sg_table_locked(&self) -> bool { + // SAFETY: Reservation ownership serializes the C table pointer. + !unsafe { (*self.as_raw_shmem()).sgt }.is_null() + } + + /// Whether backing without a Rust SG lease can be reclaimed or purged. + /// + /// Also accepts backing whose SG table was already released by idle reclaim. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 0 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that - /// borrow may outlive any lock. + /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on + /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. /// /// # Safety /// @@ -516,7 +536,9 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } + || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + { return Err(EBUSY); } self.sgt_access @@ -706,63 +728,72 @@ where { } -/// A reference to a GEM object that is known to have a mapped [`SGTable`]. -/// -/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables -/// on GEM shmem objects are revoked on driver-unbind. -/// -/// # Invariants +/// An owned backing lease, suitable for retaining copied DMA segments. /// -/// - `self.obj` always points to a valid GEM object. -/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an -/// [`SGTable`]. -/// -/// [`SGTable`]: scatterlist::SGTable -pub struct SGTableMap { - obj: NonNull>, +/// Dropping the last lease permits reclaim after device translations have been invalidated. +/// This does not itself provide access to the table: table access also requires a Bound device. +#[must_use = "retain until all device uses of the backing have ended"] +pub struct SGLease { + obj: ARef>, } -impl Deref for SGTableMap { +impl Drop for SGLease { + fn drop(&mut self) { + let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); + debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); + } +} + +/// A leased SG-table borrow. References obtained from it cannot outlive the lease. +#[must_use = "retain while using the SG table or transfer into an owned lease"] +pub struct SGTable<'a, T: DriverObject> { + table: &'a scatterlist::SGTable, + lease: SGLease, +} + +impl Deref for SGTable<'_, T> { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - // SAFETY: - // - The NonNull is guaranteed to be valid via our type invariants. - // - The sgt field is guaranteed to be initialized and valid via our type invariants. - unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + self.table } } -impl Drop for SGTableMap { - fn drop(&mut self) { - // SAFETY: `obj` is always valid via our type invariants - let obj = unsafe { self.obj.as_ref() }; - let _lock = DmaResvGuard::new(obj); - - // SAFETY: We acquired the lock needed for calling this function above - unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; +impl SGTable<'_, T> { + /// Retain the backing after copying the DMA addresses, without retaining a table borrow. + pub fn into_lease(self) -> SGLease { + self.lease } } -impl SGTableMap { - fn new(obj: &Object) -> impl Init { - // INVARIANT: - // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, - // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. - // SAFETY: - // - `obj` is fully initialized, making this function safe to call. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; +/// Device-unbind cleanup registration for locally allocated SG tables. +/// +/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may +/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. +struct SGTableCleanup { + obj: NonNull>, +} - Ok(Self { obj: obj.into() }) +impl Drop for SGTableCleanup { + fn drop(&mut self) { + // SAFETY: The registration is dropped before its GEM object. + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. + // Reclaim may already have freed the table; the C free helper requires a non-null one. + unsafe { + if !(*obj.as_raw_shmem()).sgt.is_null() { + bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); + } + } } } -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Send for SGTableMap {} -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Sync for SGTableMap {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Send for SGTableCleanup {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Sync for SGTableCleanup {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -839,6 +870,14 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: This new faux device is private to the fixture. Use direct DMA with a + // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -885,19 +924,225 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.read8(offset), expected); + assert_eq!(vmap.try_read8(offset).unwrap(), expected); } Ok(()) } - // TODO: I would love to actually test the success paths of sg_table(), but that would require - // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave - // that for someone else. + #[test] + fn sg_lease_counting() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let second = obj.sg_table(dev.as_ref().as_ref())?; + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(second); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn release_idle_without_mappings() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes these fields. + let empty = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + ) + }; + assert!(empty.0); + assert!(empty.1); + // SAFETY: Still no device mappings or leases; an already idle release succeeds too. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn release_refused_while_leased() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no actual device mappings were installed. + assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; no actual device mappings were installed. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(bind); + // SAFETY: Reservation held; the mapping lease is now the sole owner. + assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; the remaining lease must still exclude release. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; no device mapping or SG lease remains. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn purge_madvised_after_last_lease() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no real device access. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; all device uses and SG leases have ended. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the backing fields and advice. + let (empty_sgt, empty_pages, madv) = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + obj.madv_locked(), + ) + }; + assert!(empty_sgt); + assert!(empty_pages); + assert_eq!(madv, -1); + } + assert!(!obj.madvise(0)); + assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn sg_rebuild_after_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.vmap::<4>()?.as_view().write32(0x12345678, 0); + for _ in 0..3 { + let table = obj.sg_table(dev.as_ref().as_ref())?; + assert!(table.iter().count() > 0); + drop(table); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; all leases have dropped, with no device mappings. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes the table pointer. + assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); + } + assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); + } + // Drop with a rebuilt table exercises the surviving devres cleanup registration too. + drop(obj.sg_table(dev.as_ref().as_ref())?); + Ok(()) + } + + #[test] + fn purge_after_idle_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + unsafe { obj.release_idle_pages_locked()? }; + } + assert!(obj.madvise(1)); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + assert!(unsafe { obj.unleased_pages_releasable_locked() }); + // SAFETY: Reservation held; no device access or SG lease remains. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the advice and table pointer. + let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; + assert_eq!(madv, -1); + assert!(empty); + Ok(()) + } + + #[test] + fn purge_refuses_kernel_vmap() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let vmap = obj.vmap::<4>()?; + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; the vmap must prevent the backing from being freed. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + // SAFETY: Reservation held; the vmap must exclude idle release too. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + } + drop(vmap); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device, vmap or SG users remain. + unsafe { obj.purge_locked()? }; + Ok(()) + } + + #[test] + fn sg_reclaim_excludes_new_leases() -> Result { + let (_dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); + obj.sgt_access.store(SGT_USERS, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); + obj.sgt_access.store(0, Ordering::Release); + Ok(()) + } + + #[test] + fn sg_import_attachment_is_not_owned() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mut table = MaybeUninit::::zeroed(); + let mut attachment = MaybeUninit::::zeroed(); + // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay + // live until the lease drops and are detached before the real object destructor runs. + unsafe { + (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); + (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); + } + let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { + let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held. Capture results before restoring the fixture, so a + // failed assertion cannot leave a mock attachment for the GEM destructor. + unsafe { + let held = obj.release_idle_pages_locked(); + drop(lease); + ( + no_cleanup, + held, + obj.release_idle_pages_locked(), + obj.purge_locked(), + (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), + ) + } + }); + // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. + unsafe { + (*obj.as_raw()).import_attach = ptr::null_mut(); + (*obj.as_raw_shmem()).sgt = ptr::null_mut(); + } + let (no_cleanup, held, unleased, purge, unchanged) = result?; + assert!(no_cleanup); + assert_eq!(held, Err(EBUSY)); + assert_eq!(unleased, Err(EINVAL)); + assert_eq!(purge, Err(EBUSY)); + assert!(unchanged); + Ok(()) + } - // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it - // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device - // bound to it. + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -908,10 +1153,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + // A failed lookup must not bind the cleanup registration to the wrong device. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // TODO: Someday, we should test that creating an sg_table here still succeeds. + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); Ok(()) } diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs index a022e08223588b..72ba50c57286b4 100644 --- a/rust/kernel/drm/syncobj.rs +++ b/rust/kernel/drm/syncobj.rs @@ -15,6 +15,14 @@ pub struct SyncObj { } impl SyncObj { + /// Borrows the raw sync object for synchronous calls into DRM helpers. + /// The caller must not release the owned reference or retain this pointer + /// beyond the lifetime of `self` without taking another reference. + #[inline(always)] + pub fn as_raw(&self) -> *mut bindings::drm_syncobj { + self.ptr + } + /// Looks up a sync object by its handle for a given `File`. pub fn lookup_handle( file: &drm::File, diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a7f..e43d5ae131c853 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,6 +151,59 @@ impl Mm { } } +/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). +/// +/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report +/// show for a process. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RssCounter { + /// Resident file-backed pages (`MM_FILEPAGES`). + File, + /// Resident anonymous pages (`MM_ANONPAGES`). + Anon, + /// Swapped-out anonymous pages (`MM_SWAPENTS`). + Swap, + /// Resident shared-memory pages (`MM_SHMEMPAGES`). + Shmem, +} + +impl RssCounter { + fn as_raw(self) -> core::ffi::c_int { + let member = match self { + Self::File => bindings::MM_FILEPAGES, + Self::Anon => bindings::MM_ANONPAGES, + Self::Swap => bindings::MM_SWAPENTS, + Self::Shmem => bindings::MM_SHMEMPAGES, + }; + member as core::ffi::c_int + } +} + +// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these +// methods are safe even if `mm_users` is zero. +impl Mm { + /// Adds `pages` (which may be negative) to one of the resident-set counters. + /// + /// This is how a driver attributes pages it holds on behalf of a process, but which are not + /// mapped into it, to that process. Every page added must be subtracted again before the + /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad + /// rss-counter state. + #[inline] + pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; + } + + /// Returns the exact current value of one of the resident-set counters. + #[inline] + pub fn rss_counter(&self, counter: RssCounter) -> usize { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } + } +} + // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer.