From 2d8c0fe879e26583b62655534a3aba86b1dfc80f Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 16 Oct 2026 15:57:16 -0300 Subject: [PATCH 01/28] drm/asahi: mmu: make the secondary root of PublishedRoots optional A single-instance G17 firmware (the M4 Pro runs both roles on one coprocessor) has no secondary kernel root. Make secondary_root_offset an Option: the pagetables region then only needs the primary root, mirror_secondary_root() does nothing, and the TTBAT context-1 kernel alias is installed for every PublishedRoots firmware, since activate_render_context needs it. T8140 keeps Some(0x40000) and behaves as before. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/mod.rs | 5 +++-- drivers/gpu/drm/asahi/hw/t8140.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 27 +++++++++++++++------------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 953f57aaef7df..82ea402528403 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -148,8 +148,9 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, /// Crash log buffer of each [`Role`]. pub(crate) crash_buffers: [FixedBuffer; 2], /// Parameter buffer page metrics shared with the firmware. diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 7c6975b71a4c8..886794b928055 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -23,7 +23,7 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. crash_buffers: [ diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7fa207c9ff5e4..d8f3e8ebaca58 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -154,17 +154,17 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } @@ -3260,7 +3260,10 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; @@ -3372,11 +3375,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3443,7 +3446,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 From 40507bad232c7539f3c7c2540bac818a7aa90f26 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 16 Oct 2026 16:04:08 -0300 Subject: [PATCH 02/28] drm/asahi: g17: support a single firmware coprocessor The M4 Pro firmware runs both G17 roles on one coprocessor. Prepare the runtime for such a target; T8140 keeps two instances and behaves as before. - Firmware.secondary is an Option, created only when the configuration names a secondary root; boot, init, POWER_ON_DONE, the opening-retire wait, stop and drop skip it when absent. - event::Transport::roles(): event, trace and statistics service and the idle recheck cover only the instances that exist. - Config::recovery_slot_keys (T8140: SLOT_KEYS) replaces the hard-coded physical address; None leaves recovery to the firmware's own blame. - Config::gpu_generation is reported to userspace (T8140: G17). - The pm_metrics client VA comes from the target's configuration instead of hw::t8140::CONFIG in user_aliases and render preparation. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/event.rs | 8 ++- drivers/gpu/drm/asahi/g17/initdata.rs | 16 ++++- drivers/gpu/drm/asahi/g17/mod.rs | 59 ++++++++++++++----- drivers/gpu/drm/asahi/g17/recovery.rs | 4 +- drivers/gpu/drm/asahi/g17/recovery/slot.rs | 6 +- drivers/gpu/drm/asahi/g17/runtime.rs | 8 ++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 6 +- .../drm/asahi/g17/runtime/render/prepare.rs | 2 +- drivers/gpu/drm/asahi/hw/t8140.rs | 4 +- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c81..9508a827be4b1 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6b2f7c18584ef..11d410697bbb3 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -155,6 +155,10 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -235,6 +239,8 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +253,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -695,6 +705,8 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -1150,7 +1162,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1376,7 +1388,7 @@ impl super::recovery::Memory for InitData { .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 82ea402528403..bd54614a08fc5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -155,6 +155,12 @@ pub(crate) struct Config { pub(crate) crash_buffers: [FixedBuffer; 2], /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, } /// A firmware coprocessor instance. @@ -472,7 +478,9 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, rings: channel::Rings, @@ -492,7 +500,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -502,6 +512,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -965,8 +984,14 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; let bridge = Bridge::enable(regs)?; @@ -995,7 +1020,9 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } let root_entries = session.uat.adopt_firmware_root_entries()?; session.init.build( dev, @@ -1008,12 +1035,14 @@ impl Gpu { session.uat.mirror_secondary_root()?; session.bridge.regs.enable_g17_gating()?; session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?)?; + } session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; @@ -1025,15 +1054,17 @@ impl Gpu { dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1105,7 +1136,7 @@ impl gpu::Gpu for Gpu { | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); - params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index c23a5f0c13d07..89f8d33bdb80c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -13,8 +13,8 @@ mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 238b269747511..bd9d4c96be14f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index a9622d5909499..f5b682d382dd8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1466,7 +1466,9 @@ impl super::Shared { { let mut state = self.state.lock(); if let Some(firmware) = state.as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } } @@ -1490,7 +1492,9 @@ impl super::Shared { let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f8fcf253b41a5..4fd6e56741986 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,6 +24,9 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { let firmware = &mut *self.firmware; firmware @@ -445,7 +448,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 2acd464bbeea4..bce05997159e1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -36,7 +36,7 @@ impl BuildResources { metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 886794b928055..63be97432c4d2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -7,7 +7,7 @@ use crate::g17::{ FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -42,6 +42,8 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; From 750dd1a8e5902e4e7478ce9213278b2131b63266 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:52:04 -0300 Subject: [PATCH 03/28] drm/asahi: g17: describe the firmware build in a variant table G17 firmware builds that share the interface version still differ in a few queue rules. Name them in a fw::Variant that each SoC's Config points at, instead of hard-coding the A18 Pro firmware's (26A428) choices: - qid_floor: the first QID the host may use; queue admission and the UAPI queue limit follow the render pairs it leaves; - pointers: where the item-ring producer and the recovery target live in a queue's pointer block; - mmio_pair_registration: whether render pairs are registered through the MMIO queue ports; - mcache: whether render kick entries may name an MCache table; - few_primitives and ray_traversal: whether those render and compute features are offered and accepted. The variant reaches the queue-ID table, the compute and render graphs, command validation and the feature report through the structures that already carry per-device state. T8140 uses V26A428, which keeps every value it had, so its behaviour does not change. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/command.rs | 20 ++++++- drivers/gpu/drm/asahi/g17/fw/mod.rs | 3 ++ drivers/gpu/drm/asahi/g17/fw/queue.rs | 52 ++++++++++++------- drivers/gpu/drm/asahi/g17/fw/variant.rs | 48 +++++++++++++++++ drivers/gpu/drm/asahi/g17/kick.rs | 36 ++++++++++--- drivers/gpu/drm/asahi/g17/mod.rs | 33 +++++++++--- drivers/gpu/drm/asahi/g17/queue/admission.rs | 24 +++++---- drivers/gpu/drm/asahi/g17/queue/compute.rs | 31 +++++++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 13 ++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 12 ++++- .../gpu/drm/asahi/g17/queue/render/memory.rs | 19 ++++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 4 ++ .../drm/asahi/g17/queue/render/publication.rs | 2 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/runtime.rs | 21 +++++--- drivers/gpu/drm/asahi/g17/runtime/render.rs | 5 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 1 + drivers/gpu/drm/asahi/g17/validation.rs | 5 +- drivers/gpu/drm/asahi/hw/t8140.rs | 1 + 19 files changed, 263 insertions(+), 70 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 26785a326c6f0..c65f7de6ac915 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -351,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -424,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -457,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -489,6 +495,10 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -497,7 +507,7 @@ impl Payload { Ok(Validated::Compute { long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -509,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 8f08531097f87..1cb46fce2d602 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index ac46fda33eb5f..fd79e92d6eb20 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -115,26 +117,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +148,40 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); - /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Self { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target).store(u32::MAX, Ordering::Relaxed); + } + block + } + + /// The host word at `offset`, one of the two [`PointerLayout`] names. + fn word(&self, offset: usize) -> &AtomicU32 { + if offset == core::mem::offset_of!(Self, word_50) { + &self.word_50 + } else { + &self.word_40 } } @@ -174,12 +190,12 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) { self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + self.word(layout.producer).store(base, Ordering::Relaxed); } } diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 0000000000000..c3548d208390d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, +}; diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 7aaca95992f8d..f223f967b00dd 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -30,7 +30,15 @@ pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; // channels are shared after exact retirement, rather than pinned per client. const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +/// Logical queues that can each hold a render pair: what the render reserve leaves beside the +/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone +/// (every execution context with no floor). +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); assert!(COMPUTE_CHANNELS >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -65,15 +73,19 @@ pub(crate) struct QueueIds { initial_render: [Option; 2], compute_publication: u64, leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], compute_publication: 0, leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -117,7 +129,11 @@ impl QueueIds { Err(EINVAL) }; } - let initial = if engine == DataMaster::Compute && self.leased == QID_COUNT { + // The QIDs below the floor are never the host's; the reserve shrinks + // with them, so the compute channel count stays the same. + let total = QID_COUNT - self.floor; + let reserve = COMPUTE_RENDER_RESERVE - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() .copied() @@ -145,16 +161,22 @@ impl QueueIds { // Include that transition when protecting render's minimum, and // count the replacement as available compute capacity. let replacement = usize::from(initial.is_some()); - if QID_COUNT - self.leased + replacement - <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) + if total - self.leased + replacement + <= reserve.saturating_sub(render - replacement) { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { @@ -220,7 +242,7 @@ mod accounting_tests { use super::*; fn table() -> KBox { - KBox::init(QueueIds::new(), GFP_KERNEL).unwrap() + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() } #[test] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index bd54614a08fc5..ad8cad427c3f0 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -161,6 +161,8 @@ pub(crate) struct Config { pub(crate) recovery_slot_keys: Option, /// GPU generation reported to userspace. pub(crate) gpu_generation: hw::GpuGen, + /// The firmware build this SoC runs. + pub(crate) variant: &'static fw::Variant, } /// A firmware coprocessor instance. @@ -487,6 +489,7 @@ struct Firmware { recovery: recovery::State, effort: power::Effort, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, /// Render kick ring storage, kept and reused for the life of the device. @@ -631,6 +634,7 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -896,6 +900,7 @@ impl Gpu { dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant: cfg.variant, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -1007,7 +1012,12 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: cfg.variant, + queues <- runtime::Registry::new( + shared.clusters, + shared.descriptor_flags, + shared.variant, + ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, @@ -1132,10 +1142,17 @@ impl gpu::Gpu for Gpu { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + let variant = self.cfg.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1176,7 +1193,7 @@ impl gpu::Gpu for Gpu { fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1215,8 +1232,12 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.cfg.variant, + )?; let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 7fc4d79a5f95c..dd908aac48c01 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -360,14 +360,20 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, } impl Pool { - pub(crate) fn new() -> Result> { + pub(crate) fn new(max_queues: u32) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, }), GFP_KERNEL, ) @@ -379,7 +385,7 @@ impl Pool { pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { + if count >= self.max_queues { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( @@ -615,7 +621,7 @@ mod accounting_tests { #[test] fn logical_queue_count_returns_on_drop() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); for _ in 0..MAX_EXECUTION_CONTEXTS { leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); @@ -732,7 +738,7 @@ impl Pool { impl Pool { pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS)?; + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; let state = self.state.lock(); let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { bits | if *owner != 0 { 1u64 << slot } else { 0 } @@ -751,7 +757,7 @@ mod fault_capacity_tests { #[test] fn closed_faulting_clients_return_leases_after_drain() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); for owner in 1..=512 { let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); @@ -774,7 +780,7 @@ mod fault_capacity_tests { #[test] fn published_exit_kill_holds_survive_close_and_drain() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); pool.release_owner(1); @@ -816,7 +822,7 @@ mod deadline_tests { #[test] fn all_pairs_busy_no_progress_expires_without_release() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); { let mut state = pool.state.lock(); // No pair has room, including the requesting owner's retained pair. @@ -835,7 +841,7 @@ mod deadline_tests { #[test] fn busy_pair_observes_cancellation_without_a_release_event() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); { let mut state = pool.state.lock(); for slot in 1..cfg::RENDER_SLOTS { @@ -855,7 +861,7 @@ mod deadline_tests { #[test] fn cancellation_precedes_available_capacity() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), Err(error) if error == ECANCELED)); assert_eq!(pool.state.lock().active, 0); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9ccdd703106b7..c2ec11a1b16fd 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,7 +9,9 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{ + channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*, variant::PointerLayout, + }, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -113,10 +115,12 @@ struct Graph { records: KernelObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new(alloc: &Allocator<'_>, context: &Context, pointers: PointerLayout) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; @@ -141,7 +145,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers); view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; @@ -160,6 +164,7 @@ impl Graph { records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, }) } @@ -221,7 +226,7 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) @@ -276,7 +281,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(POINTERS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -530,9 +537,10 @@ impl Queue { backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, &context, pointers)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -1607,14 +1615,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(POINTERS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(POINTERS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1706,7 +1717,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -2130,7 +2141,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 84d1bd5a2fd71..6a9b15789339f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -56,10 +56,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +77,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,6 +92,7 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], @@ -109,6 +117,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -121,6 +130,7 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), @@ -334,6 +344,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index b69cd4d565736..dbdaf33c93493 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -155,6 +155,7 @@ pub(crate) struct Pair { graph_started: bool, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -200,6 +201,7 @@ impl Pair { first: &Arc, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -240,6 +242,7 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { kick_aliases: Some([ta_alias, fragment_alias]), @@ -256,6 +259,7 @@ impl Pair { graph_started: false, clusters, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -426,9 +430,14 @@ impl Pair { }, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -1719,6 +1728,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 53bc3cfbd6aeb..ab2a3b8327054 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -94,6 +94,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +103,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +157,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers); Ok(()) })?; } @@ -166,6 +169,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +192,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +214,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -380,10 +386,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 5b3f2a00bad6d..02e606b2d175e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -26,6 +26,7 @@ pub(crate) struct ParkedPair { ordinal: u64, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, @@ -91,6 +92,7 @@ impl Pair { ordinal: self.ordinal, clusters: self.clusters, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, @@ -167,6 +169,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -230,6 +233,7 @@ impl ParkedPair { initial_packet: next.initial_packet, clusters: self.clusters, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 72d36afcf5b87..310784bb4e1f0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -86,7 +86,7 @@ impl<'a> Lane<'a> { }; Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec6..f718cd47f90cf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0); } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index f5b682d382dd8..bd70090ac160d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -72,16 +72,17 @@ impl Registry { pub(super) fn new( clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, + admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -410,9 +411,10 @@ pub(super) enum Rebind { #[kunit_tests(asahi_g17_uninstalled_cleanup)] mod uninstalled_cleanup_tests { use super::*; + use crate::g17::fw::variant::V26A428; #[test] fn empty_reservation_prevents_reuse_until_explicit_cancellation() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let first = registry.reserve_compute(1).unwrap(); assert!(registry.reserved(first).is_ok()); assert!(registry.reserve_compute(1).is_err()); @@ -871,7 +873,13 @@ impl Backend { // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. self.compute_scheduler(&alloc, &lease, admission)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + let queue = compute::Queue::new( + &alloc, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -1643,10 +1651,11 @@ impl Backend { #[kunit_tests(asahi_g17_shared_capacity)] mod shared_capacity_tests { use super::*; + use crate::g17::fw::variant::V26A428; #[test] fn all_logical_render_owners_fit_beside_shared_compute() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); // Physical compute channels are device-owned and bounded independently // of how many logical clients have queued unsignalled work. @@ -1666,7 +1675,7 @@ mod shared_capacity_tests { #[test] fn four_clients_have_capacity_before_any_owner_closes() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); for client in 0..4u64 { for queue in 0..9u64 { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 88340442bbcae..a399a1532550f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -527,6 +527,11 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index bce05997159e1..18c6abf294fc4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -275,6 +275,7 @@ impl super::super::Backend { first, resources.config.clusters, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef..b2a6cf6e8bfe3 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -90,10 +90,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +108,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 63be97432c4d2..d4a887000cf6e 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -44,6 +44,7 @@ pub(crate) const CONFIG: Config = Config { }, recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, + variant: &crate::g17::fw::variant::V26A428, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; From 9038ef8d328cef14e038513ccaf55dc03869006b Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:53:05 -0300 Subject: [PATCH 04/28] drm/asahi: g17: take the per-SoC tables from Config The initialization builder names hw::t8140 for the register windows, the reserved register-map slots, the performance-table configuration and the operating-point count. Move these references into Config so a second G17 SoC can supply its own tables, and make the crash log placement an enum (CrashBuffers::Fixed) that a SoC whose firmware places its own crash log can extend. T8140 passes its existing tables; nothing changes for it. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/initdata.rs | 22 +++++++++++++--------- drivers/gpu/drm/asahi/g17/mod.rs | 23 ++++++++++++++++++++--- drivers/gpu/drm/asahi/hw/t8140.rs | 9 +++++++-- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 11d410697bbb3..f98b96686e86a 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -716,7 +716,7 @@ impl InitData { base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - self.map_register_windows(&env)?; + self.map_register_windows(&env, cfg.windows)?; let tables = self.build_descriptor_tables(&env)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; @@ -725,8 +725,12 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { let (phys, va, size) = window.mapping().ok_or(EINVAL)?; self.aliases.push( env.vm.map_io( @@ -1000,13 +1004,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1192,7 +1196,7 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + fn opps(dev: &AsahiDrmDevice, count: usize) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; let table = node .parse_phandle(c_str!("operating-points-v2"), 0) @@ -1207,7 +1211,7 @@ impl InitData { GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.len() != count { return Err(EINVAL); } Ok(opps) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ad8cad427c3f0..6b9aa8691e066 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -151,10 +151,18 @@ pub(crate) struct Config { /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for /// a single-instance firmware. pub(crate) secondary_root_offset: Option, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -165,6 +173,13 @@ pub(crate) struct Config { pub(crate) variant: &'static fw::Variant, } +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), +} + /// A firmware coprocessor instance. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Role { @@ -346,7 +361,9 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; + let buffer = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => &buffers[index], + }; let mut crash_buffer = KernelObject::new( dev, uat.kernel_lower_vm(), diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index d4a887000cf6e..6e4368bf9da77 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,6 +4,7 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; @@ -26,7 +27,7 @@ pub(crate) const CONFIG: Config = Config { secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,7 +36,7 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { @@ -44,6 +45,10 @@ pub(crate) const CONFIG: Config = Config { }, recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, variant: &crate::g17::fw::variant::V26A428, }; From 24d1e5f4d8d31dd320048546cd53230a61203a9f Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:54:00 -0300 Subject: [PATCH 05/28] drm/asahi: g17: operating points with one voltage per cluster The performance tables of the hardware data have sixteen voltage lanes per state. The A18 Pro lists one voltage per operating point, which the host broadcasts to every lane of table A and stores in lane 0 of table B. GPUs with several clusters have one voltage per cluster, listed as one opp-microvolt cell per cluster (as the T6031 device tree already does). Read every cell, and store cluster n's voltage in lane n of both tables, leaving the other lanes zero. A single cell keeps the current broadcast, so the T8140 tables are unchanged. All operating points must have the same number of cells. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 36 +++++++++++++++--------- drivers/gpu/drm/asahi/g17/initdata.rs | 12 ++++++-- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 2b61ecc118320..cf37c124cde52 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -676,14 +676,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +733,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +763,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index f98b96686e86a..07e6ee4306899 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1203,15 +1203,23 @@ impl InitData { .ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != count { + if opps.len() != count || opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) From 1ef423a08b3cbc81856afbe3c5854ac25b0893b6 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:54:43 -0300 Subject: [PATCH 06/28] drm/asahi: g17: register windows with several instances A register-map slot of the hardware data can describe several instances of a block: the firmware finds instance n at va + n * size, and the entry's total size covers all of them while its element size is that of one instance. On SoCs with several memory controllers or AFR blocks the instances are not contiguous in physical memory. Give RegisterWindow an instance count and a physical stride, map each instance separately, and fill total_size and element_size from them. Every T8140 window has one instance, so its register map and mappings are unchanged. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 37 +++++++++++++++--------- drivers/gpu/drm/asahi/g17/initdata.rs | 24 ++++++++------- drivers/gpu/drm/asahi/hw/t8140.rs | 2 ++ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index cf37c124cde52..384d2e6072c57 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -591,31 +591,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va + let start = va & !mask; + let span = va .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +640,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +661,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 07e6ee4306899..568602f08d241 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -731,17 +731,19 @@ impl InitData { windows: &[abi::RegisterWindow], ) -> Result { for window in windows { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 6e4368bf9da77..ebff08a4fa475 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -75,6 +75,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } From f23df96e697bc05a47029efd4fe786f900b08a83 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:55:58 -0300 Subject: [PATCH 07/28] drm/asahi: g17: let the variant choose the init acknowledgement The A18 Pro firmware acknowledges the initialization root with an INITDATA_ACK message and expects POWER_ON_DONE once both coprocessors are initialized. Other firmware builds acknowledge by setting word +0x14 of the primary status A to 1 and send no message. Add fw::variant::InitAck and Variant::power_on_done. With InitAck::Status the host polls status A for up to 3 s after handing over the primary root, failing early if the firmware crashes; the opening-record wait that follows is unchanged. POWER_ON_DONE is sent only where the variant asks for it. V26A428 keeps the message and POWER_ON_DONE, so T8140 is unchanged. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/fw/variant.rs | 16 +++++++ drivers/gpu/drm/asahi/g17/initdata.rs | 5 +++ drivers/gpu/drm/asahi/g17/mod.rs | 55 +++++++++++++++++++++---- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index c3548d208390d..62283fa87a88d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -18,8 +18,22 @@ pub(crate) struct PointerLayout { pub(crate) recovery_target: Option, } +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + /// The layouts and rules of one G17 firmware build. pub(crate) struct Variant { + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, /// First QID the host may use; the firmware keeps the lower ones for itself. pub(crate) qid_floor: usize, /// Host words of a queue's pointer block. @@ -36,6 +50,8 @@ pub(crate) struct Variant { /// The A18 Pro firmware (26A428). pub(crate) const V26A428: Variant = Variant { + init_ack: InitAck::Message, + power_on_done: true, qid_floor: 0, pointers: PointerLayout { producer: 0x40, diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 568602f08d241..ef74533990c2c 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1273,6 +1273,11 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( Self::status_offset(role) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6b9aa8691e066..d168a20427115 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -220,6 +220,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -429,8 +432,9 @@ impl Coprocessor { Pin::new(&mut self.rtkit).boot() } - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { let mut rtkit = Pin::new(&mut self.rtkit); rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; rtkit.as_mut().start_endpoint(EP_DOORBELL)?; @@ -438,7 +442,10 @@ impl Coprocessor { EP_FIRMWARE, MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), )?; - self.state.wait_for_ack() + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. @@ -1061,13 +1068,19 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; session.uat.mirror_secondary_root()?; session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - if let Some(secondary) = session.secondary.as_mut() { - secondary.init(session.init.root_va(Role::Secondary)?)?; + let ack = cfg.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } - session.primary.notify(MSG_POWER_ON_DONE)?; if let Some(secondary) = session.secondary.as_mut() { - secondary.notify(MSG_POWER_ON_DONE)?; + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if cfg.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } } Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session @@ -1076,6 +1089,32 @@ impl Gpu { Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, From 39e990b44482f46fb6bed6c3b7eb2413e0f0331a Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:58:12 -0300 Subject: [PATCH 08/28] drm/asahi: mmu: firmware tables in no-map reserved memory The UAT code reaches the firmware's root and kernel-window tables through struct page, so it requires the pagetables region to be part of the kernel's memory map; it refuses a root in no-map memory ("DT using no-map?"). On the M4 Pro the boot loader reserves the UAT ranges as no-map regions outside /memory, and the firmware also builds tables in the l2-pagetables region. Add mmu::UatQuirks::reserved_tables: the pagetables and l2-pagetables regions are mapped write-back once (pgtable::ReservedTables), and every table access goes through with_table(), which uses that mapping for a page in either region and struct page otherwise. The CPU's cached lines of both regions are cleaned and invalidated after the firmware booted, so the host reads what the firmware wrote. This is a temporary path: once the boot loader adds the UAT ranges to /memory, as m1n1 does, the regions lose no-map and this commit can be reverted. T8140 and the older GPUs use UatQuirks::DEFAULT and are unchanged. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g16/uat.rs | 2 +- drivers/gpu/drm/asahi/g17/mod.rs | 5 + drivers/gpu/drm/asahi/gpu.rs | 1 + drivers/gpu/drm/asahi/hw/t8140.rs | 1 + drivers/gpu/drm/asahi/mmu.rs | 65 ++++++- drivers/gpu/drm/asahi/pgtable.rs | 277 +++++++++++++++++++++--------- 6 files changed, 263 insertions(+), 88 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/uat.rs b/drivers/gpu/drm/asahi/g16/uat.rs index 132d7efea6883..54104712c294c 100644 --- a/drivers/gpu/drm/asahi/g16/uat.rs +++ b/drivers/gpu/drm/asahi/g16/uat.rs @@ -74,7 +74,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index d168a20427115..306526cac710b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -163,6 +163,8 @@ pub(crate) struct Config { pub(crate) perf: &'static fw::initdata::PerfConfig, /// Number of operating points the device tree lists. pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -997,6 +999,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -1057,6 +1060,8 @@ impl Gpu { if let Some(secondary) = session.secondary.as_mut() { secondary.boot()?; } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); let root_entries = session.uat.adopt_firmware_root_entries()?; session.init.build( dev, diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e9e207f0ad3d8..b68381ce60b30 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -747,6 +747,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index ebff08a4fa475..8e998062b2c77 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -49,6 +49,7 @@ pub(crate) const CONFIG: Config = Config { reserved_slots: &RESERVED_REGISTER_SLOTS, perf: &PERF, opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, variant: &crate::g17::fw::variant::V26A428, }; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index d8f3e8ebaca58..cd446e1f7c3eb 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -168,6 +168,25 @@ pub(crate) enum UatFirmware { }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; @@ -1639,6 +1658,8 @@ struct UatInner { /// caches. Set only by [`Uat::new_adopted`]. #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: bool, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] @@ -1949,6 +1970,7 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? @@ -2865,6 +2887,7 @@ impl Uat { failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3224,7 +3247,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -3244,7 +3267,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -3267,11 +3290,12 @@ impl Uat { let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3288,6 +3312,14 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { @@ -3315,7 +3347,11 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -3332,6 +3368,7 @@ impl Uat { failed: AtomicBool::new(false), #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: false, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3362,12 +3399,26 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 38c60052debce..292a0a48fd76a 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -96,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -123,14 +123,108 @@ pub(crate) fn poc_barrier() { unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; } +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) +} + /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -138,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -301,6 +398,8 @@ pub(crate) struct UatPageTable { ttb_owned: bool, /// Ownership could not be relinquished to firmware; physical tables remain until reboot. retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -323,6 +422,7 @@ impl UatPageTable { ttb, ttb_owned: true, retained: false, + reserved: None, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, @@ -372,6 +472,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -387,7 +488,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -399,6 +502,7 @@ impl UatPageTable { ttb, ttb_owned: false, retained: false, + reserved, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, @@ -483,6 +587,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -526,55 +631,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); } - })?; + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -610,13 +718,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -624,7 +729,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -659,27 +769,34 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { - // SAFETY: The live page-table page contains aligned u64 PTEs. - // with_pointer_into_page bounds this access to one PTE, and - // the parent remains allocated throughout the traversal. + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; From 54eccc50cc770bdbe9306be4337c548f399e2eb8 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 22:59:27 -0300 Subject: [PATCH 09/28] drm/asahi: mmu: per-SoC UAT cache, TLB and handoff policy The M4 Pro GPU has only ever run with the UAT policy the Gravity Linux G16 driver uses for it: every page-table and context-table change is written back (dc cvac), TLB ranges are invalidated one page at a time, the handoff page carries only the host header (current slot 0xffff_ffff, the lock never taken) and GPU-only client mappings are stored as AP 0 with the access in PXN/UXN. Whether the M4 Pro needs any of this has not been tested; the A18 Pro needs none of it. Add one UatQuirks field per setting, so a SoC can stay on the policy it was tested with and each setting can be moved to the A18 Pro's value on its own after a test: - clean_tables: clean every page-table change to the point of coherency, with the page-table cleaning the G16 backend already uses (clean_poc, no longer limited to G16), and also the handoff page and each context-table slot after it changes; - page_tlbi: invalidate page by page (vae1os), or the whole ASID above 2048 pages, instead of rvae1os; - handoff_lock and handoff_slot: whether the legacy handoff lock is taken and the current slot written at initialization; - native_gpu_permissions: the AP 0 form for GPU-only mappings of client and kernel-lower tables, on map and on reprotect. The root clean of clean_poc goes through the reserved-table mapping, so it also covers a firmware root in no-map memory. UatQuirks::DEFAULT keeps the A18 Pro policy, so T8140, the older GPUs and the adopted G15/G16 UAT are unchanged. Based on findings by Gravity Linux (Cody Ho, Niklas Sheth). Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/mem.rs | 34 ++++++++++ drivers/gpu/drm/asahi/mmu.rs | 104 +++++++++++++++++++++++++------ drivers/gpu/drm/asahi/pgtable.rs | 70 +++++++++++++++------ 3 files changed, 168 insertions(+), 40 deletions(-) diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 31c41221967fa..91fd5092aaefc 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,35 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for this GPU. +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,6 +104,10 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; @@ -167,6 +197,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index cd446e1f7c3eb..f981b1c88496c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -178,12 +178,32 @@ pub(crate) struct UatQuirks { /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the /// regions lose `no-map` and this path goes away. pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, } impl UatQuirks { /// The A18 Pro and earlier policy. pub(crate) const DEFAULT: UatQuirks = UatQuirks { reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, }; } @@ -1041,7 +1061,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1049,8 +1069,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1390,10 +1411,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1609,9 +1630,33 @@ struct UatShared { map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1655,9 +1700,10 @@ struct UatInner { /// An uncertain mapping/root is retained; future M4 allocation and publication fail. failed: AtomicBool, /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches. Set only by [`Uat::new_adopted`]. - #[cfg(CONFIG_DRM_ASAHI_G16)] + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). reserved: Option>, #[pin] @@ -1724,7 +1770,7 @@ impl HandoffGuard<'_> { impl Drop for HandoffGuard<'_> { fn drop(&mut self) { if let Self::Legacy(shared) = self { - shared.handoff().unlock(); + shared.handoff_unlock(); } } } @@ -1758,7 +1804,7 @@ impl UatInner { let shared = self.lock(); self.check_m4()?; if !self.m4_adopted { - shared.handoff().lock(); + shared.handoff_lock(); return Ok(HandoffGuard::Legacy(shared)); } let transport = M4Transport { @@ -1833,9 +1879,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1974,8 +2020,8 @@ impl Vm { )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - #[cfg(CONFIG_DRM_ASAHI_G16)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2771,6 +2817,7 @@ impl Drop for VmInner { uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } drop(transaction); @@ -2888,6 +2935,7 @@ impl Uat { // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2903,6 +2951,7 @@ impl Uat { map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3107,6 +3156,7 @@ impl Uat { uat_inner.binding_published[idx] = false; uat_inner.owned_user_roots[idx] = ttb; binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone @@ -3206,6 +3256,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3324,20 +3375,21 @@ impl Uat { /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3350,6 +3402,7 @@ impl Uat { fn make_inner( dev: driver::AsahiDevice, firmware: UatFirmware, + quirks: UatQuirks, reserved: Option>, ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; @@ -3360,14 +3413,15 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, m4_adopted: false, failed: AtomicBool::new(false), - #[cfg(CONFIG_DRM_ASAHI_G16)] - clean_poc: false, + clean_poc: quirks.clean_tables, + quirks, reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") @@ -3384,6 +3438,7 @@ impl Uat { map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3403,6 +3458,7 @@ impl Uat { ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let reserved = if quirks.reserved_tables { let mut tables = pgtable::ReservedTables::new(); @@ -3417,7 +3473,7 @@ impl Uat { None }; - let inner = Self::make_inner(dev.clone(), firmware, reserved)?; + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); @@ -3481,11 +3537,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3506,8 +3566,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 292a0a48fd76a..05b0c4ad28715 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -378,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -400,6 +416,8 @@ pub(crate) struct UatPageTable { retained: bool, /// Reserved regions holding tables this page table may reach. reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -409,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: bool, } @@ -423,45 +440,47 @@ impl UatPageTable { ttb_owned: true, retained: false, reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G16), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a /// `dsb sy`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without G16 support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_G16)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_G16))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -503,11 +522,11 @@ impl UatPageTable { ttb_owned: false, retained: false, reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: false, }) } @@ -897,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -916,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -942,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); From 1baee970e0a9d5bf0343f72eed61badfc5918159 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:00:33 -0300 Subject: [PATCH 10/28] dt-bindings: mailbox: apple,ascwrap-v6: Add the T6040 The M4 Pro (T6040) GPU coprocessor is an ASCWrap v6 like the A18 Pro's: the ADT node /arm-io/gfx-asc is compatible with iop,ascwrap-v6, its wrapper is 0x88000 bytes with the ASC mailbox at +0x8000, and its vector-base register follows the same layout. Unlike the A18 Pro it runs a single firmware instance, with the GFX role. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index 4798b62bd8430..09b3b45b4ed92 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -4,20 +4,23 @@ $id: http://devicetree.org/schemas/mailbox/apple,ascwrap-v6.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 ASCWrap v6 mailbox +title: Apple ASCWrap v6 GPU mailbox maintainers: - Ryan Murray description: - Mailbox interface of the T8140 GPU coprocessors. The wrapper resource - contains the coprocessor control registers and mailbox FIFOs. A separate - resource provides the coprocessor vector-base register. The firmware role - identifies which of the two GPU firmware instances uses the mailbox. + Mailbox interface of the GPU coprocessors of the T8140 and T6040. The + wrapper resource contains the coprocessor control registers and mailbox + FIFOs. A separate resource provides the coprocessor vector-base register. + The firmware role identifies which GPU firmware instance uses the mailbox; + the T6040 runs a single one. properties: compatible: - const: apple,t8140-ascwrap-v6 + enum: + - apple,t6040-ascwrap-v6 + - apple,t8140-ascwrap-v6 reg: items: @@ -57,6 +60,17 @@ required: - '#mbox-cells' - apple,firmware-role +allOf: + - if: + properties: + compatible: + contains: + const: apple,t6040-ascwrap-v6 + then: + properties: + apple,firmware-role: + const: GFX + additionalProperties: false examples: From 29a913a5c222f2bec2ecae62a4dc4af223e5962d Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:01:29 -0300 Subject: [PATCH 11/28] soc: apple: mailbox: support the T6040 ASCWrap v6 The ASCWrap v6 provider accepts only the T8140's GPU coprocessor windows. The M4 Pro (T6040) GPU coprocessor is the same block with one firmware role, GFX, at wrapper 0x28a600000 and vector base 0x28a050000 (ADT /arm-io/gfx-asc). Replace the hard-coded T8140 checks with a role table per compatible: each role has its name, its firmware role and its fixed CPU-physical windows, which the probe validates as before. Add the apple,t6040-ascwrap-v6 compatible with its single role. T8140 keeps its two roles at the same addresses. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/soc/apple/mailbox.c | 84 +++++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index e8faed1ec56e6..a85e5b585f319 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -60,6 +60,8 @@ #define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL #define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL #define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL +#define APPLE_T6040_GFX_WRAPPER_BASE 0x28a600000ULL +#define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -116,9 +118,19 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_G17_MBOX_RX_BUDGET 64 #define APPLE_G17_MBOX_ATOMIC_TX_US 1000 +/* One firmware role of an ASCWrap v6 provider, at its fixed CPU-physical windows. */ +struct apple_mbox_ascwrap_v6_role_desc { + const char *name; + enum apple_mbox_ascwrap_v6_role role; + resource_size_t wrapper_base; + resource_size_t iop_vbar_base; +}; + struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + const struct apple_mbox_ascwrap_v6_role_desc *ascwrap_v6_roles; + unsigned int num_ascwrap_v6_roles; bool ap_initializes_mailboxes; unsigned int control_full; @@ -675,6 +687,8 @@ static int apple_mbox_probe(struct platform_device *pdev) int ret; char *irqname; const char *firmware_role; + const struct apple_mbox_ascwrap_v6_role_desc *role; + unsigned int i; struct apple_mbox *mbox; struct device *dev = &pdev->dev; struct resource *res, *iop_vbar_res; @@ -691,7 +705,7 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->hw->is_ascwrap_v6) { /* * Validate the entire provider identity before mapping either - * resource. These are fixed T8140 CPU-physical windows and role + * resource. These are fixed per-SoC CPU-physical windows and role * assignments, not generic consumer-provided aliases. */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); @@ -709,28 +723,26 @@ static int apple_mbox_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "missing ASCWrap v6 firmware role\n"); - if (!strcmp(firmware_role, "GFX")) { - mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; - if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) - return dev_err_probe(dev, -EINVAL, - "GFX ASCWrap v6 resources do not match T8140\n"); - } else if (!strcmp(firmware_role, "GFX1")) { - mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; - if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) - return dev_err_probe(dev, -EINVAL, - "GFX1 ASCWrap v6 resources do not match T8140\n"); - } else { + for (i = 0; i < mbox->hw->num_ascwrap_v6_roles; i++) { + role = &mbox->hw->ascwrap_v6_roles[i]; + if (!strcmp(firmware_role, role->name)) + break; + } + if (i == mbox->hw->num_ascwrap_v6_roles) return dev_err_probe(dev, -EINVAL, "invalid ASCWrap v6 firmware role\n"); - } + if (res->start != role->wrapper_base || + iop_vbar_res->start != role->iop_vbar_base) + return dev_err_probe(dev, -EINVAL, + "%s ASCWrap v6 resources do not match the SoC\n", + role->name); + mbox->ascwrap_v6.role = role->role; mbox->ascwrap_v6.wrapper_start = res->start; mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* T8140 implements both the CPU start and the CPU stop. */ + /* T8140 and T6040 implement both the CPU start and the CPU stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = @@ -874,9 +886,44 @@ static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { .has_irq_controls = false, }; -static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8140_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T8140_GFX_WRAPPER_BASE, APPLE_T8140_GFX_IOP_VBAR_BASE }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_T8140_GFX1_WRAPPER_BASE, APPLE_T8140_GFX1_IOP_VBAR_BASE }, +}; + +/* The T6040 runs a single GPU firmware instance. */ +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6040_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T6040_GFX_WRAPPER_BASE, APPLE_T6040_GFX_IOP_VBAR_BASE }, +}; + +static const struct apple_mbox_hw apple_mbox_t8140_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t8140_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t8140_ascwrap_v6_roles), + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + +static const struct apple_mbox_hw apple_mbox_t6040_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t6040_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t6040_ascwrap_v6_roles), .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -929,7 +976,8 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, - { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, + { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, + { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* From d717a601430aea6711ffc7486f5edb16ca7e7f66 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:03:58 -0300 Subject: [PATCH 12/28] dt-bindings: gpu: apple,agx: Add apple,agx-t6040 The M4 Pro (T6040) GPU is described like the A18 Pro's: an asc and an sgx register range, its GPU power domain and an operating-point table with one voltage per cluster. It has a single GFX coprocessor mailbox, operating-point tables for its CS and AFR clock domains (apple,cs-opp and apple,afr-opp, the names the older Apple GPUs' device trees use), and two more memory regions than the A18 Pro: fw-data, the loaded firmware's data segment, where the firmware places its crash log, and l2-pagetables, where the firmware builds the page tables of its kernel window. The boot loader may report the macOS version whose boot firmware loaded the GPU firmware in apple,firmware-version, which identifies the firmware build. The memory-region lists differ per compatible, so move them into the per-compatible branches; the existing compatibles keep the same regions and names. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- .../devicetree/bindings/gpu/apple,agx.yaml | 145 +++++++++++++----- 1 file changed, 110 insertions(+), 35 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 0118e3afa27af..843d7c4af8f9a 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-t6040 - apple,agx-t8132 - apple,agx-t8140 - items: @@ -51,8 +52,12 @@ properties: - description: Region containing a shared handoff structure for VM management coordination - - description: Power calibration blob or firmware shared-L2 page - - description: GPU calibration blob or preserved firmware text + - description: + Power calibration blob, firmware shared-L2 page or firmware data + segment + - description: + GPU calibration blob, preserved firmware text or firmware + kernel-window page tables - description: Shared global variables or preserved firmware data memory-region-names: @@ -61,8 +66,8 @@ properties: - const: ttbs - const: pagetables - const: handoff - - enum: [hw-cal-a, shared-l2] - - enum: [hw-cal-b, fw-text] + - enum: [hw-cal-a, shared-l2, fw-data] + - enum: [hw-cal-b, fw-text, l2-pagetables] - enum: [globals, fw-data] apple,pmp: @@ -71,6 +76,26 @@ properties: operating-points-v2: true + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + maxItems: 3 + description: + macOS version (major, minor, patch) whose boot firmware loaded the GPU + firmware, which identifies the firmware build. Filled in by the boot + loader. + + apple,cs-opp: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Operating-point table of the GPU's CS clock domain, one voltage cell per + state. + + apple,afr-opp: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Operating-point table of the GPU's AFR clock domain, one voltage cell + per state. + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -121,20 +146,45 @@ allOf: else: properties: apple,pmp: false + - if: + properties: + compatible: + contains: + const: apple,agx-t8140 + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + items: + - description: Region containing GPU MMU TTBs + - description: Region containing GPU MMU page tables + - description: + Region containing a shared handoff structure for VM + management coordination memory-region-names: - minItems: 3 items: - const: ttbs - const: pagetables - const: handoff - - const: hw-cal-a - - const: hw-cal-b - - const: globals + required: + - power-domains + - operating-points-v2 - if: properties: compatible: contains: - const: apple,agx-t8140 + const: apple,agx-t6040 then: properties: reg: @@ -148,39 +198,64 @@ allOf: mboxes: items: - description: GFX coprocessor mailbox - - description: GFX1 coprocessor mailbox memory-region: - maxItems: 3 + items: + - description: Region containing GPU MMU TTBs + - description: Region containing GPU MMU page tables + - description: + Region containing a shared handoff structure for VM + management coordination + - description: + Data segment of the loaded GPU firmware, where the firmware + places its crash log + - description: + Region in which the GPU firmware builds the page tables of + its kernel window memory-region-names: - maxItems: 3 + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: fw-data + - const: l2-pagetables required: - power-domains - operating-points-v2 - else: - if: + - apple,cs-opp + - apple,afr-opp + - if: + not: properties: compatible: - not: - contains: - const: apple,agx-t8132 - then: - properties: - reg: - items: - - description: GPU coprocessor control registers - - description: GPU block MMIO registers - reg-names: - items: - - const: asc - - const: sgx - mboxes: - maxItems: 1 - memory-region: - minItems: 6 - memory-region-names: - minItems: 6 - required: - - apple,firmware-abi + contains: + enum: + - apple,agx-t6040 + - apple,agx-t8132 + - apple,agx-t8140 + then: + properties: + reg: + items: + - description: GPU coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + maxItems: 1 + memory-region: + minItems: 6 + memory-region-names: + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: hw-cal-a + - const: hw-cal-b + - const: globals + required: + - apple,firmware-abi additionalProperties: false From abb36e8aa3203f4cb28c1b2498d636617d01111d Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:15:11 -0300 Subject: [PATCH 13/28] drm/asahi: g17: 25G83 initialization layout The M4 Pro runs the macOS 26.6 GPU firmware (25G83, mBoot-18000.161.10). It has the A18 Pro firmware's interface version and the same initialization objects, with these differences: - One coprocessor carries both roles: one main configuration with the secondary-only words (0x300 = 4, the power-configuration pointer), a zeroed page at unk_18, 0x32c and 0x344 zero, and one merged root naming status B, the power state and the configuration. - The cluster is 0xd4000 bytes: status B at 0x10000 is 0xf180 bytes long, status A follows it, and the power state is its own page, of which only the fast-die target and release temperatures are set. - The hardware data (HwData25G83) is the A18 Pro's up to 0x2554 with 8 bytes inserted before 0x2570 and 4 more before 0x2614, so the completion rings move to 0x2620 and the QoS pointer to 0x26b8. It has the CS and AFR performance tables at 0x1a48 and 0x1b90 and carries one voltage lane per cluster. - The power configuration (PowerConfig25G83) is at bundle +0x5380 and has two blocks fewer than the A18 Pro's; the auxiliary data and an AON temperature-source table follow it. - Region C has other values, now RegionCConfig in Config (T8140 keeps its values). - The five views and the firmware area are separate pages. fw::variant::InitGraph selects the graph. The 25G83 contents are constants except for the operating points (the device tree's GPU, CS and AFR tables), the core count, three pointers and a random word. They were derived by evaluating, for the J616s, the hardware-data, region C and power encoders that Gravity Linux (Niklas Sheth, Cody Ho) wrote for this firmware, which the M4 Pro has run with since; no code of theirs is carried. At boot the built hardware data, power configuration and region C are hashed (FNV-1a, without the runtime words) and compared with the hashes of the encoders' output, and the GPU is not started if they differ. The leakage coefficients in the power configuration are this unit's values. They are not in the ADT, and whether they are per SoC or per device is not known; they stay a driver constant until that is. Where the 25G83 graph deviates from the A18 Pro's is only known from that setup; every value is the one that runs today. Based on findings by Gravity Linux (Cody Ho, Niklas Sheth). Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 82 ++- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 681 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/variant.rs | 12 + drivers/gpu/drm/asahi/g17/initdata.rs | 102 ++- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 238 ++++++ drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/hw/t8140.rs | 1 + 7 files changed, 1069 insertions(+), 50 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 384d2e6072c57..3092e599caa84 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -1190,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1218,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1234,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1245,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1255,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1306,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1340,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1385,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1432,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 0000000000000..c237896a49fb1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 62283fa87a88d..caba54f590f53 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -27,8 +27,19 @@ pub(crate) enum InitAck { Status, } +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + /// The layouts and rules of one G17 firmware build. pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, /// How the firmware acknowledges its initialization. pub(crate) init_ack: InitAck, /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor @@ -50,6 +61,7 @@ pub(crate) struct Variant { /// The A18 Pro firmware (26A428). pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, init_ack: InitAck::Message, power_on_done: true, qid_floor: 0, diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index ef74533990c2c..e73f2f5bbaf4c 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -159,6 +161,11 @@ pub(super) struct InitData { pm_metrics_va: u64, /// `Config::recovery_slot_keys`. recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -241,6 +248,9 @@ impl InitData { metrics_fw_va: 0, pm_metrics_va: 0, recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -512,7 +522,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -532,7 +542,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -611,7 +621,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -704,6 +714,7 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + active_cores: u32, ) -> Result { self.pm_metrics_va = cfg.pm_metrics.va; self.recovery_slot_keys = cfg.recovery_slot_keys; @@ -716,8 +727,11 @@ impl InitData { base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; + if cfg.variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } self.map_register_windows(&env, cfg.windows)?; - let tables = self.build_descriptor_tables(&env)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -749,13 +763,17 @@ impl InitData { } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -1020,7 +1038,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1068,8 +1086,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1125,7 +1143,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1199,10 +1217,18 @@ impl InitData { } fn opps(dev: &AsahiDrmDevice, count: usize) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { // One cell for the whole GPU, or one per cluster. @@ -1221,22 +1247,22 @@ impl InitData { GFP_KERNEL, )?; } - if opps.len() != count || opps.iter().any(|opp| opp.cells != opps[0].cells) { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1244,7 +1270,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1280,7 +1306,7 @@ impl InitData { pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1292,7 +1318,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1367,7 +1393,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1382,8 +1408,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1394,17 +1424,17 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; @@ -1422,21 +1452,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1448,17 +1478,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1468,7 +1502,7 @@ impl super::recovery::Memory for InitData { fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); let mut entries = super::recovery::Entries::default(); for index in 0..256 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 0000000000000..bfb7cf1af3ecd --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 306526cac710b..3e5ba9c51d293 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -165,6 +165,8 @@ pub(crate) struct Config { pub(crate) opps: usize, /// UAT policy where it differs from the A18 Pro's. pub(crate) uat_quirks: mmu::UatQuirks, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -1069,6 +1071,7 @@ impl Gpu { &session.uat, &session._render_global, &session._pm_metrics, + shared.active_cores, )?; session.uat.confirm_firmware_root_entries(root_entries)?; session.uat.mirror_secondary_root()?; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 8e998062b2c77..b3dd5d1bf851b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -50,6 +50,7 @@ pub(crate) const CONFIG: Config = Config { perf: &PERF, opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, variant: &crate::g17::fw::variant::V26A428, }; From 0d9efa515188d93cd775e240b20bf4ad29a258c8 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:16:51 -0300 Subject: [PATCH 14/28] drm/asahi: g17: render lead commands for the 25G83 firmware The 25G83 firmware expects two commands in a render pair's item rings that the A18 Pro firmware does not take: - InitBufferManager (tag 5) before the first tiling descriptor that uses a buffer manager, naming its UAT context, buffer slot, PB state and tiling stamp; - FragmentBarrier (tag 4) before every fragment descriptor, waiting for the paired tiling stamp in addition to the kick-entry dependency. Add Variant::lead_commands. With it, render publication writes these commands after each stage's QueueConfig record in its configuration slot and publishes them as the first item of the stage, and the buffer manager remembers that it was initialized once a tiling pass with InitBufferManager was published. V26A428 has no lead commands. Based on findings by Gravity Linux (Cody Ho, Niklas Sheth). Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/buffer.rs | 12 +++ drivers/gpu/drm/asahi/g17/fw/queue.rs | 93 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/variant.rs | 4 + drivers/gpu/drm/asahi/g17/queue/render.rs | 16 +++- .../gpu/drm/asahi/g17/queue/render/memory.rs | 52 +++++++++++ .../drm/asahi/g17/queue/render/publication.rs | 30 +++--- 6 files changed, 195 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index c4b07c603fb82..8f3a03fbb8dba 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index fd79e92d6eb20..ace7f3712c4df 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -96,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -541,6 +545,95 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp. The G16 driver writes it as the 25G83 Metal driver does +/// ("emitBarrierCommands"), in addition to the kick-entry dependency. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u32, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id: fragment_id.into(), + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index caba54f590f53..9d7c4f5cd720c 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -57,6 +57,9 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, } /// The A18 Pro firmware (26A428). @@ -73,4 +76,5 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + lead_commands: false, }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index dbdaf33c93493..eb4f380a51351 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -556,6 +556,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -688,6 +695,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -697,6 +705,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -783,7 +792,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index ab2a3b8327054..34a8e992e207c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -516,6 +519,9 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { @@ -609,6 +615,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id as u32, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 310784bb4e1f0..9f4cb05b6bede 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,18 +73,25 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, From f8dc19862078a7ad72cd3e10fe22052a02609e76 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:21:23 -0300 Subject: [PATCH 15/28] drm/asahi: g17: 25G83 render and compute commands The 25G83 firmware takes render descriptors with other register programs, and the M4 Pro's two clusters of ten cores change the buffers whose layout follows the core layout. Firmware-build data (Variant::programs, RegisterPrograms::V25G83): - Tiling array of 83 entries (checkpoints 81/83/83, free-list slot index 79): the A18 Pro's without 0x1748 and 0x1740, with the twelve tiler parameter sub-buffer registers and 0x10131 = 0x88. - Main fragment array of 91 entries (checkpoints 87/88/88, slot index 84): with 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068. - Both drop the SCS enable 0x017d9 unless the stage has shader-context scratch; a stage with it carries 0x017d9 = 1 as on the A18 Pro, and the counts and indices after it move by one. - Tile-state base 0x36011 instead of 0x3717f: with the A18 Pro's bits a fragment pass more than 80 tiles wide and with more than 4096 tiles never completes, which includes every full-screen pass on the 3456x2234 panel. - The TPC object also holds the seven tiler parameter sub-buffers the tiling registers name. Core-layout data: - The deflake buffer's sub-buffers follow each cluster's control words: +0x20/+0x2a0 with one cluster, +0xa0/+0x820 with two, where the viewport words at +0x900 would land in the third sub-buffer and are not written (render::Deflake). - The compute operand-state buffer holds two banks of three records per core and one tail per cluster: 0x44000 bytes with banks of 0x1e000 at a stride of 0x1e400 for two clusters of ten cores (compute::OperandState). Only the measured layouts are accepted. V26A428 and one cluster keep the A18 Pro's programs and buffers. The 25G83 register lists come from the register programs that the Gravity Linux G16 driver (Niklas Sheth, Cody Ho) runs on the M4 Pro; the code is new. Based on findings by Gravity Linux (Cody Ho, Niklas Sheth). Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/buffer/target.rs | 61 ++++- drivers/gpu/drm/asahi/g17/fw/compute.rs | 77 ++++-- drivers/gpu/drm/asahi/g17/fw/queue.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 247 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/fw/variant.rs | 3 + drivers/gpu/drm/asahi/g17/mod.rs | 4 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 21 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 18 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 36 ++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 11 files changed, 413 insertions(+), 61 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c93..bfa64c0b27a6a 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block (the 25G83 Metal driver's default parameter-buffer sizes): +/// the TPC and seven sub-buffers the tiling registers name. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 3564c3d776a51..717e3d25a4962 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -152,13 +152,52 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster (the A18 Pro's six cores). + const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; + + /// The layout for `clusters` clusters with `cores` enabled cores in all. + pub(crate) fn new(clusters: u32, cores: u32) -> Result<&'static OperandState> { + // shortcut: only the two measured layouts; add others when a GPU with them is supported. + match (clusters, cores) { + (1, _) => Ok(&Self::ONE_BY_SIX), + (2, 20) => Ok(&Self::TWO_BY_TEN), + _ => Err(ENODEV), + } + } +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +322,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +346,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -349,12 +391,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x10428, key), RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +406,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index ace7f3712c4df..5d9a6202943f3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -546,8 +546,8 @@ impl QueueConfig { } /// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work -/// waits for the paired tiling stamp. The G16 driver writes it as the 25G83 Metal driver does -/// ("emitBarrierCommands"), in addition to the kick-entry dependency. +/// waits for the paired tiling stamp, in addition to the kick-entry dependency, as the 25G83 +/// Metal driver writes it ("emitBarrierCommands"). #[repr(C, packed)] #[derive(Copy, Clone, Debug)] pub(crate) struct FragmentBarrier { diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 33c42a5a2aca7..f28ef376b887d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,59 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// The layout for a GPU with `clusters` clusters. + pub(crate) fn new(clusters: u32) -> Result { + match clusters { + 1 => Ok(Self { + sub: [0x20, 0x2a0], + viewport: true, + }), + // shortcut: only one and two clusters were measured; add other counts when a GPU + // with them is supported. + 2 => Ok(Self { + sub: [0xa0, 0x820], + viewport: false, + }), + _ => Err(EINVAL), + } + } +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +153,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +315,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +370,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -491,9 +551,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; + let [deflake_20, deflake_2a0] = args.deflake.sub; let deflake = u64::from(user_offset32(r.deflake_va)?); - let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)?); - let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -581,6 +642,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +772,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -902,7 +1091,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,7 +1109,11 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { let mut trimmed = [registers[0]; TA_REGISTERS]; @@ -945,7 +1139,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -980,11 +1177,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1371,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,7 +1454,12 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; @@ -1294,7 +1504,10 @@ impl FragmentSlot { desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 9d7c4f5cd720c..1d4ea097d78a8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -60,6 +60,8 @@ pub(crate) struct Variant { /// Render item rings carry lead commands: InitBufferManager before the first tiling pass /// of a buffer manager, and FragmentBarrier before every fragment descriptor. pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, } /// The A18 Pro firmware (26A428). @@ -77,4 +79,5 @@ pub(crate) const V26A428: Variant = Variant { few_primitives: true, ray_traversal: true, lead_commands: false, + programs: super::render::RegisterPrograms::G17P, }; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3e5ba9c51d293..51b39836470c5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -663,6 +663,8 @@ struct Shared { clusters: u32, active_cores: u32, variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -922,6 +924,7 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let operand_state = fw::compute::OperandState::new(id.num_clusters, active_cores)?; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -929,6 +932,7 @@ impl Gpu { clusters: id.num_clusters, active_cores, variant: cfg.variant, + operand_state, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c2ec11a1b16fd..bf8d7ffa6afb4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -10,7 +10,10 @@ use crate::g17::{ channel::Rings, context::Context, fw::{ - channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*, variant::PointerLayout, + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, }, object::{Allocator, CpuMap, KernelObject}, }; @@ -117,17 +120,24 @@ struct Graph { reclaimed: u32, /// Host words of the pointer block in this firmware build. pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context, pointers: PointerLayout) -> Result { + fn new( + alloc: &Allocator<'_>, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; @@ -165,6 +175,7 @@ impl Graph { record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, + operand_state, }) } @@ -538,9 +549,10 @@ impl Queue { owner: u64, context: Arc, pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers)?; + let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -1823,6 +1835,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index eb4f380a51351..66c515f6f59de 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -226,7 +226,7 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; let (fragment, fragment_alias) = @@ -360,14 +360,17 @@ impl Pair { let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = abi::Deflake::new(self.clusters)?; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -397,6 +400,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -428,6 +432,8 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; // Without an MCache table kick-entry bit 58 stays clear. @@ -665,6 +671,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -674,6 +681,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -1010,7 +1018,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -1726,7 +1734,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 34a8e992e207c..09ccb689b6632 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -319,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 02e606b2d175e..ce77d870de641 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -153,7 +153,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index bd70090ac160d..fad13c8baaee6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -879,6 +879,7 @@ impl Backend { self.owner, self.context.clone(), self.shared.variant.pointers, + self.shared.operand_state, )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); From 20e51646e036432fa9498a24e5330b8d02068612 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Thu, 22 Oct 2026 23:24:53 -0300 Subject: [PATCH 16/28] drm/asahi: g17: add the T6040 (M4 Pro) Add the M4 Pro GPU to the G17 runtime. Its firmware (25G83) runs on one ASCWrap v6 coprocessor with the GFX role, which the mailbox provider starts and stops; the GPU power domains are ordinary PMGR domains. The ID registers report one die of two clusters with ten cores each, family 0xa, variant 3, revision B1. - hw/t6040.rs: the configuration, the 25 register windows (the memory controller and AFR slots with four instances each), the performance table configuration and region C. - fw::variant::V25G83: QIDs from 32, the producer at pointer block +0x50 without a recovery target, no MMIO render-pair registration, no MCache, lead commands, the 25G83 register programs and initialization graph, acknowledgment in status A and no POWER_ON_DONE. Few-primitives renders hang this firmware and a stalled ray traversal wedges the GPU without a recovery, so neither is offered nor accepted. - CrashBuffers::FirmwareData: the firmware asks for its crash log at a physical address inside its data segment (the fw-data reserved region); the driver maps that range and RTKit echoes the request. - Config::adopt_root_entries and Config::fabric_bridge: the M4 Pro borrows the firmware root as the firmware left it and writes neither the fabric bridge nor the gating register, as it has always run. - The UABI reports the GPU as G16, which userspace drives it as. - The firmware build is identified, not inferred from the SoC: Config::firmware (fw::variant::Firmware) lists the builds a SoC supports by the macOS version whose boot firmware loaded them, which the boot loader reports in apple,firmware-version on the GPU node (as m1n1 writes it). The T6040 accepts macOS 26.6.2 (iBoot mBoot-18000.161.10, the 25G83 firmware) and refuses anything else before touching the GPU, naming what it found; the build it accepts is logged. The T8140 keeps its single unidentified build. Where the values differ from the A18 Pro's they are the ones the M4 Pro runs with today. The UAT quirks other than the no-map tables, the root adoption, the bridge and gating, the MMIO registration and POWER_ON_DONE were never tested the A18 Pro's way on this chip, and each stays a separate setting until it is. Based on findings by Gravity Linux (Cody Ho, Niklas Sheth). Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/driver.rs | 4 + drivers/gpu/drm/asahi/g17/fw/variant.rs | 45 ++++++ drivers/gpu/drm/asahi/g17/initdata.rs | 3 +- drivers/gpu/drm/asahi/g17/mod.rs | 182 +++++++++++++++++++----- drivers/gpu/drm/asahi/g17/runtime.rs | 21 +++ drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t6040.rs | 174 ++++++++++++++++++++++ drivers/gpu/drm/asahi/hw/t8140.rs | 4 +- 8 files changed, 395 insertions(+), 40 deletions(-) create mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d0dfc840265da..c26fe1671f7cf 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -146,6 +146,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 1d4ea097d78a8..bb30925f54160 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,6 +64,26 @@ pub(crate) struct Variant { pub(crate) programs: super::render::RegisterPrograms, } +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + /// The A18 Pro firmware (26A428). pub(crate) const V26A428: Variant = Variant { graph: InitGraph::G17P, @@ -81,3 +101,28 @@ pub(crate) const V26A428: Variant = Variant { lead_commands: false, programs: super::render::RegisterPrograms::G17P, }; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index e73f2f5bbaf4c..6057e7d0ac957 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -715,6 +715,7 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, active_cores: u32, + variant: &super::fw::Variant, ) -> Result { self.pm_metrics_va = cfg.pm_metrics.va; self.recovery_slot_keys = cfg.recovery_slot_keys; @@ -727,7 +728,7 @@ impl InitData { base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - if cfg.variant.graph == super::fw::variant::InitGraph::V25G83 { + if variant.graph == super::fw::variant::InitGraph::V25G83 { return self.build_25g83(&env, cfg, metrics, active_cores); } self.map_register_windows(&env, cfg.windows)?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 51b39836470c5..9f92a74da53cc 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -59,6 +59,7 @@ use kernel::{ device::{self, Core}, drm::gem::shmem, io::{ + self, IoBase, IoSysMap, Region, // @@ -165,6 +166,13 @@ pub(crate) struct Config { pub(crate) opps: usize, /// UAT policy where it differs from the A18 Pro's. pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, /// The SoC's values of region C. pub(crate) region_c: &'static fw::initdata::RegionCConfig, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is @@ -173,8 +181,8 @@ pub(crate) struct Config { pub(crate) recovery_slot_keys: Option, /// GPU generation reported to userspace. pub(crate) gpu_generation: hw::GpuGen, - /// The firmware build this SoC runs. - pub(crate) variant: &'static fw::Variant, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, } /// Where a G17 target's crash log buffers live. @@ -182,6 +190,9 @@ pub(crate) enum CrashBuffers { /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel /// lower root, mapped by the driver before the coprocessors start. Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -237,7 +248,19 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// `size` bytes at `offset` of the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData { + mem: io::mem::Mem, + offset: usize, + size: usize, + }, } impl rtkit::Buffer for CrashLog { @@ -246,7 +269,16 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData { mem, offset, size } => { + // SAFETY: shmem_map() checked that the range lies inside the mapping. + let base = unsafe { mem.ptr().add(*offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, *size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } + } } } @@ -255,8 +287,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -335,13 +367,28 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); + } + // The firmware places its buffers in its data segment, at their physical address. + let node = data.shared.dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { return Err(EINVAL); } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData { mem, offset, size }, }) } } @@ -354,7 +401,7 @@ struct Coprocessor { running: bool, rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -368,29 +415,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = match &cfg.crash_buffers { - CrashBuffers::Fixed(buffers) => &buffers[index], + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), }; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -488,6 +538,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -900,6 +958,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -931,7 +990,7 @@ impl Gpu { dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, - variant: cfg.variant, + variant, operand_state, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), @@ -974,6 +1033,42 @@ impl Gpu { Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -1032,7 +1127,10 @@ impl Gpu { }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -1045,7 +1143,7 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), registration_power_release_pending: false, - variant: cfg.variant, + variant: shared.variant, queues <- runtime::Registry::new( shared.clusters, shared.descriptor_flags, @@ -1068,7 +1166,10 @@ impl Gpu { } // The firmware built its root and kernel-window tables while booting. session.uat.invalidate_reserved_tables(); - let root_entries = session.uat.adopt_firmware_root_entries()?; + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, @@ -1076,11 +1177,16 @@ impl Gpu { &session._render_global, &session._pm_metrics, shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - let ack = cfg.variant.init_ack; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; session.primary.init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; @@ -1088,7 +1194,7 @@ impl Gpu { if let Some(secondary) = session.secondary.as_mut() { secondary.init(session.init.root_va(Role::Secondary)?, ack)?; } - if cfg.variant.power_on_done { + if shared.variant.power_on_done { session.primary.notify(MSG_POWER_ON_DONE)?; if let Some(secondary) = session.secondary.as_mut() { secondary.notify(MSG_POWER_ON_DONE)?; @@ -1212,7 +1318,7 @@ impl gpu::Gpu for Gpu { | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); - let variant = self.cfg.variant; + let variant = self.shared.variant; if variant.few_primitives { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); @@ -1304,7 +1410,7 @@ impl gpu::Gpu for Gpu { priority, usc_exec_base, self.geometry.user_usable_range(), - self.cfg.variant, + self.shared.variant, )?; let policy = scheduling_policy(priority)?; let admission_started = now_ns(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index fad13c8baaee6..cc3e4b3044b03 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1674,6 +1674,27 @@ mod shared_capacity_tests { // published identity was cancelled or retyped to achieve the fit. } + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init(Registry::new(2, [0; 2], &V25G83), GFP_KERNEL).unwrap(); + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + #[test] fn four_clients_have_capacity_before_any_owner_closes() { let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0b..2650131a4a293 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 0000000000000..db0c33fff8485 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index b3dd5d1bf851b..f5ed7bc80963b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -51,7 +51,9 @@ pub(crate) const CONFIG: Config = Config { opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, region_c: &crate::g17::fw::initdata::REGION_C_26A428, - variant: &crate::g17::fw::variant::V26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; From 92d02170df1ecf2273d13ba63c0cedbb891996d3 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 23 Oct 2026 01:29:53 -0300 Subject: [PATCH 17/28] arm64: dts: apple: t6040: Add the GPU Describe the M4 Pro GPU as the A18 Pro's is described: - pmgr_gfx: the PMGR block of the GPU power domains (ADT pmgr devices AFR 0x181 at +0x100 and GFX 0x182 at +0x110 with parent AFR), laid out as on the T6031. Both are always on: the GPU firmware keeps running and does not survive GFX going off. GPX (0x180 at +0x0) is left alone, as the boot loader leaves it on and nothing uses it. - gfx_mbox: the ASCWrap v6 coprocessor mailbox with the GFX role (ADT gfx-asc), its send-empty and receive-not-empty interrupts. - gpu: the asc and sgx ranges, the mailbox and the GFX power domain. - The PMP's GPU report entry (pmp_gfx) gets the GFX domain as its parent, as on the T8140. On the J616s the GPU firmware runs from boot on, with GFX always on, so the board claims pmp_gfx always-on: the PMP sees the GPU on whenever it is powered. Without the GPU node the entry stays unclaimed (the PMP crash of a claim with nothing powering the GPU, fixed by commit c4722f38cdcc). On the J616s, add its sixteen operating points in performance-state order with one opp-microvolt cell per cluster (ADT /arm-io/sgx perf-states and perf-states1), the eight of its CS and AFR clock domains (cs-perf-states, afr-perf-states), and the memory the boot loader reserves for the GPU firmware: its data segment (fw-data), where it places its crash log, and the UAT ranges (ttbs, pagetables, l2-pagetables, handoff). The operating points are binned per device: these are this unit's values until the boot loader fills them in. The UAT ranges are no-map until the boot loader adds them to /memory. Binding: apple,agx-t6040 and apple,t6040-ascwrap-v6, earlier in this series. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 206 ++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 22 +++ arch/arm64/boot/dts/apple/t6040.dtsi | 40 ++++- 3 files changed, 266 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 7c8280cd45850..9360fad43afa2 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -71,6 +71,40 @@ isp_heap: isp-heap { iommu-addresses = <&isp 0x100 0x0 0x0 0x2000000>; }; + + /* + * Data segment of the loaded GPU firmware (ADT sgx gfx-data-*); it + * places its crash log here. + */ + gpu_fw_data: gpu-fw-data@10001d44000 { + reg = <0x100 0x01d44000 0x0 0x138000>; + no-map; + }; + + /* + * The UAT ranges (ADT sgx gpu-region-*, gfx-shared-region-*, + * gfx-shared-l2-region-*, gfx-handoff-*). They stay no-map until + * the boot loader adds them to /memory. + */ + uat_ttbs: uat-ttbs@10bfffb8000 { + reg = <0x10b 0xfffb8000 0x0 0x4000>; + no-map; + }; + + uat_pagetables: uat-pagetables@10bfff78000 { + reg = <0x10b 0xfff78000 0x0 0x40000>; + no-map; + }; + + uat_l2_pagetables: uat-l2-pagetables@10bfff74000 { + reg = <0x10b 0xfff74000 0x0 0x4000>; + no-map; + }; + + uat_handoff: uat-handoff@10bfff70000 { + reg = <0x10b 0xfff70000 0x0 0x4000>; + no-map; + }; }; memory@0 { @@ -90,6 +124,158 @@ }; }; + /* + * GPU performance states, one opp-microvolt cell per cluster (ADT + * /arm-io/sgx perf-states and perf-states1). The operating points are + * binned per device: these are this unit's values until the boot + * loader fills them in. Entries keep the order of the GPU performance + * states, which is not sorted by frequency. + */ + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000 125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <605000 585000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <655000 640000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <690000 690000>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <715000 725000>; + }; + opp05 { + opp-hz = /bits/ 64 <952000000>; + opp-microvolt = <755000 770000>; + }; + opp06 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <755000 770000>; + }; + opp07 { + opp-hz = /bits/ 64 <1062000000>; + opp-microvolt = <795000 810000>; + }; + opp08 { + opp-hz = /bits/ 64 <1182000000>; + opp-microvolt = <795000 810000>; + }; + opp09 { + opp-hz = /bits/ 64 <1182000000>; + opp-microvolt = <845000 860000>; + }; + opp10 { + opp-hz = /bits/ 64 <1312000000>; + opp-microvolt = <845000 860000>; + }; + opp11 { + opp-hz = /bits/ 64 <1242000000>; + opp-microvolt = <900000 915000>; + }; + opp12 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <900000 915000>; + }; + opp13 { + opp-hz = /bits/ 64 <1326000000>; + opp-microvolt = <945000 960000>; + }; + opp14 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <945000 960000>; + }; + opp15 { + opp-hz = /bits/ 64 <1578000000>; + opp-microvolt = <1030000 1045000>; + }; + }; + + /* The GPU's CS clock domain (ADT cs-perf-states) */ + gpu_cs_opp: opp-table-gpucs { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <24000000>; + opp-microvolt = <555000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <555000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <620000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <660000>; + }; + opp04 { + opp-hz = /bits/ 64 <992000000>; + opp-microvolt = <715000>; + }; + opp05 { + opp-hz = /bits/ 64 <1188000000>; + opp-microvolt = <780000>; + }; + opp06 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <870000>; + }; + opp07 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <910000>; + }; + }; + + /* The GPU's AFR clock domain (ADT afr-perf-states) */ + gpu_afr_opp: opp-table-gpuafr { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <24000000>; + opp-microvolt = <555000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <555000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <620000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <660000>; + }; + opp04 { + opp-hz = /bits/ 64 <992000000>; + opp-microvolt = <715000>; + }; + opp05 { + opp-hz = /bits/ 64 <1188000000>; + opp-microvolt = <780000>; + }; + opp06 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <870000>; + }; + opp07 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <910000>; + }; + }; + /* * CPU die temperature: SMC key TCMz (CPU die max), sensor 0 of * smc_hwmon. The PMP's fast die-temperature limit regulates (TCMz @@ -618,3 +804,23 @@ &isp_dart2 { status = "okay"; }; + +&gfx_mbox { + status = "okay"; +}; + +&gpu { + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&gpu_fw_data>, <&uat_l2_pagetables>; + memory-region-names = "ttbs", "pagetables", "handoff", "fw-data", + "l2-pagetables"; + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + status = "okay"; +}; + +/* The GPU firmware runs from boot on (ps_gfx is always on): the PMP must see the GPU on. */ +&pmp_gfx { + apple,always-on; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index fc0228c236be6..46f7491e234bc 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -7,6 +7,28 @@ * Copyright The Asahi Linux Contributors */ +&pmgr_gfx { + ps_afr: power-controller@100 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afr"; + apple,always-on; + }; + + /* The GPU firmware keeps running and does not survive this domain going off. */ + ps_gfx: power-controller@110 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_afr>; + apple,always-on; + }; +}; + &pmgr { ps_ispsens0: power-controller@0 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index e5a87f1331d54..d623687d2a7e9 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -608,6 +608,40 @@ power-domains = <&ps_avd_sys>; }; + /* Named after its sgx range: the coprocessor range is the mailbox node's too. */ + gpu: gpu@288000000 { + compatible = "apple,agx-t6040"; + reg = <0x2 0x8a600000 0x0 0x88000>, + <0x2 0x88000000 0x0 0x3758000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>; + power-domains = <&ps_gfx>; + status = "disabled"; + }; + + pmgr_gfx: power-management@288e80000 { + compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x88e80000 0x0 0x4000>; + /* child nodes are added in t6040-pmgr.dtsi */ + }; + + gfx_mbox: mailbox@28a600000 { + compatible = "apple,t6040-ascwrap-v6"; + reg = <0x2 0x8a600000 0x0 0x88000>, + <0x2 0x8a050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + disp0_dart: iommu@402300000 { compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x02300000 0x0 0xc000>; @@ -868,8 +902,9 @@ * The GPU (soc-device 13). Without it the running PMP treats * the GPU as off and holds it at its lowest state, but claiming * it while nothing powers the GPU crashes the PMP firmware - * (SError) right after it starts. The GPU node takes it as a - * power domain, so it is on only while the GPU driver runs. + * (SError) right after it starts. It sits on the GPU's own + * power state, and a board that powers the GPU (ps_gfx stays on + * while the GPU firmware runs) claims it always-on. */ pmp_gfx: report@d { compatible = "apple,t6041-pmp-v2-report-entry", @@ -877,6 +912,7 @@ reg = <0xd>; label = "pmp-gfx"; #power-domain-cells = <0>; + power-domains = <&ps_gfx>; }; }; From b147e1f460b56e398bc085b2457b1ba163cf4505 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 23 Oct 2026 16:23:56 -0300 Subject: [PATCH 18/28] arm64: dts: apple: t6040-j616s: Add the gpu alias The boot loader finds the GPU node through the gpu alias, as m1n1's dt_set_gpu() does on every other Apple SoC (t600x, t602x, t603x, t8103, t8112, t8122, t8132, t8140 all have it). Without it a loader cannot fill in apple,firmware-version on the T6040 GPU node, and the driver refuses the GPU unless the device tree carries the property itself. Fixes: 92d02170df1e ("arm64: dts: apple: t6040: Add the GPU") Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index a63293d4b5602..76bc9e1adb406 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -26,6 +26,8 @@ /* used by the loader to hand over the display */ dcp = &dcp; disp0 = &display; + /* the boot loader finds the GPU node through this alias */ + gpu = &gpu; }; chosen { From 45ac35fa279642f98ea7326144716755fb3a39a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:37:06 +1100 Subject: [PATCH 19/28] drm/asahi: g17: move the fw-data crash log mapping to its own file Commit 20e51646e036 ("drm/asahi: g17: add the T6040 (M4 Pro)") maps the crash log that the 25G83 firmware places in its data segment the way Gravity Linux's M4 GPU driver maps its crash buffer (CrashBuffer and BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs, written by Cody Ho). It looks up the same reserved region by name, applies the same bounds checks against it and maps it write-back the same way. The code derives from Gravity Linux commit bd25d194b0a2 ("drm/asahi: add asynchronous M4 GPU prototype driver"), which is GPL-2.0-only, so it cannot stay in g17/mod.rs, which is GPL-2.0-only OR MIT. Move the mapping into g17/data_segment.rs under GPL-2.0-only, with a header that credits Cody Ho and Gravity Linux and links the original commit. The behaviour is unchanged. Originally-by: Cody Ho Link: https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/data_segment.rs | 56 +++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 33 ++++--------- 2 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs new file mode 100644 index 0000000000000..02bcc66de8f51 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/data_segment.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2026 The Gravity Linux Contributors +// +// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and +// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. + +//! Crash log buffers in the GPU firmware's data segment. +//! +//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it +//! by physical address inside its data segment, the `fw-data` reserved region, which the boot +//! loader keeps for the loaded firmware until reboot. + +use kernel::{ + c_str, device, + io::{ + self, + IoSysMap, + Region, // + }, + prelude::*, +}; + +/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. +pub(super) struct DataSegmentLog { + mem: io::mem::Mem, + offset: usize, + size: usize, +} + +impl DataSegmentLog { + /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which + /// must lie inside the `fw-data` region of `dev`. + pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { + return Err(EINVAL); + } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + Ok(Self { mem, offset, size }) + } + + /// The CPU view of the buffer. + pub(super) fn buf(&mut self) -> Result> { + // SAFETY: map() checked that the range lies inside the mapping. + let base = unsafe { self.mem.ptr().add(self.offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9f92a74da53cc..779dc250f3feb 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -23,6 +23,7 @@ mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; +mod data_segment; mod dependency; mod event; mod feed; @@ -59,7 +60,6 @@ use kernel::{ device::{self, Core}, drm::gem::shmem, io::{ - self, IoBase, IoSysMap, Region, // @@ -255,12 +255,8 @@ pub(crate) struct CrashLog { enum CrashLogMap { /// The driver's buffer ([`CrashBuffers::Fixed`]). Object(shmem::VMapOwned), - /// `size` bytes at `offset` of the firmware's data segment ([`CrashBuffers::FirmwareData`]). - FirmwareData { - mem: io::mem::Mem, - offset: usize, - size: usize, - }, + /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData(data_segment::DataSegmentLog), } impl rtkit::Buffer for CrashLog { @@ -271,13 +267,7 @@ impl rtkit::Buffer for CrashLog { fn buf(&mut self) -> Result> { match &mut self.map { CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), - CrashLogMap::FirmwareData { mem, offset, size } => { - // SAFETY: shmem_map() checked that the range lies inside the mapping. - let base = unsafe { mem.ptr().add(*offset) }; - let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, *size)?; - // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. - Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) - } + CrashLogMap::FirmwareData(log) => log.buf(), } } } @@ -377,18 +367,13 @@ impl rtkit::Operations for RoleOps { }); } // The firmware places its buffers in its data segment, at their physical address. - let node = data.shared.dev.of_node().ok_or(EINVAL)?; - let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; - let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; - if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { - return Err(EINVAL); - } - // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps - // it until reboot; only the requested subrange is exposed. - let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; Ok(CrashLog { iova, - map: CrashLogMap::FirmwareData { mem, offset, size }, + map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( + &data.shared.dev, + iova, + size, + )?), }) } } From fd21bf6faed6f978223e61444424ee918c199838 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:38:59 +1100 Subject: [PATCH 20/28] drm/asahi: g17: take the core-layout buffers from Config The operand-state and deflake buffer layouts were chosen from the cluster and core counts the GPU reports at probe, so the A18 Pro kept its layouts only as long as it reported one cluster. Name them in the SoC's Config instead. The T8140 gets the layouts it has always used, with no new check. The T6040 names the layouts measured on the 20-core M4 Pro and the counts they were measured on, and a GPU that reports other counts is refused at probe, as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/fw/compute.rs | 16 ++------- drivers/gpu/drm/asahi/g17/fw/render.rs | 26 ++++++--------- drivers/gpu/drm/asahi/g17/mod.rs | 33 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 5 ++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 3 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 29 +++++++++++++--- drivers/gpu/drm/asahi/g17/runtime/render.rs | 8 ++++- .../drm/asahi/g17/runtime/render/prepare.rs | 1 + drivers/gpu/drm/asahi/hw/t6040.rs | 8 ++++- drivers/gpu/drm/asahi/hw/t8140.rs | 6 ++++ 10 files changed, 96 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 717e3d25a4962..0cb7304e30677 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -167,8 +167,8 @@ pub(crate) struct OperandState { } impl OperandState { - /// One cluster (the A18 Pro's six cores). - const ONE_BY_SIX: OperandState = OperandState { + /// One cluster of six cores (A18 Pro). + pub(crate) const ONE_BY_SIX: OperandState = OperandState { r10229: 0x12800, r140a8: 0x13000, r10099: 0x9405, @@ -178,7 +178,7 @@ impl OperandState { size: 0x14000, }; /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. - const TWO_BY_TEN: OperandState = OperandState { + pub(crate) const TWO_BY_TEN: OperandState = OperandState { r10229: 0x3c800, r140a8: 0x3d800, r10099: 0x1e405, @@ -187,16 +187,6 @@ impl OperandState { r0a5c9: 0x1e000, size: 0x44000, }; - - /// The layout for `clusters` clusters with `cores` enabled cores in all. - pub(crate) fn new(clusters: u32, cores: u32) -> Result<&'static OperandState> { - // shortcut: only the two measured layouts; add others when a GPU with them is supported. - match (clusters, cores) { - (1, _) => Ok(&Self::ONE_BY_SIX), - (2, 20) => Ok(&Self::TWO_BY_TEN), - _ => Err(ENODEV), - } - } } /// Checkpoints into the main register array, as entry counts: after the diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index f28ef376b887d..0e270d40392a8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -60,22 +60,16 @@ pub(crate) struct Deflake { } impl Deflake { - /// The layout for a GPU with `clusters` clusters. - pub(crate) fn new(clusters: u32) -> Result { - match clusters { - 1 => Ok(Self { - sub: [0x20, 0x2a0], - viewport: true, - }), - // shortcut: only one and two clusters were measured; add other counts when a GPU - // with them is supported. - 2 => Ok(Self { - sub: [0xa0, 0x820], - viewport: false, - }), - _ => Err(EINVAL), - } - } + /// One cluster (A18 Pro). + pub(crate) const ONE_CLUSTER: Deflake = Deflake { + sub: [0x20, 0x2a0], + viewport: true, + }; + /// Two clusters (M4 Pro). + pub(crate) const TWO_CLUSTERS: Deflake = Deflake { + sub: [0xa0, 0x820], + viewport: false, + }; } /// The register programs of a firmware build's render descriptors. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 779dc250f3feb..3906e8c19ff26 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -183,6 +183,19 @@ pub(crate) struct Config { pub(crate) gpu_generation: hw::GpuGen, /// The firmware builds this SoC runs. pub(crate) firmware: fw::variant::Firmware, + /// Layouts of the command buffers that follow the GPU's core layout. + pub(crate) core_layout: CoreLayout, +} + +/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. +pub(crate) struct CoreLayout { + /// The cluster and enabled core counts the layouts were measured on, which the GPU must + /// report at probe; `None` where the layouts hold for every configuration of the SoC. + pub(crate) cores: Option<(u32, u32)>, + /// A compute queue's operand-state buffer. + pub(crate) operand_state: fw::compute::OperandState, + /// A render's deflake buffer. + pub(crate) deflake: fw::render::Deflake, } /// Where a G17 target's crash log buffers live. @@ -708,6 +721,8 @@ struct Shared { variant: &'static fw::Variant, /// Layout of compute queues' operand-state buffers for this GPU's cores. operand_state: &'static fw::compute::OperandState, + /// Layout of renders' deflake buffers for this GPU's cores. + deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -968,7 +983,19 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); - let operand_state = fw::compute::OperandState::new(id.num_clusters, active_cores)?; + if cfg + .core_layout + .cores + .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + { + dev_err!( + dev.as_ref(), + "Unsupported GPU core layout: {} clusters with {} cores\n", + id.num_clusters, + active_cores + ); + return Err(ENODEV); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -976,7 +1003,8 @@ impl Gpu { clusters: id.num_clusters, active_cores, variant, - operand_state, + operand_state: &cfg.core_layout.operand_state, + deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -1131,6 +1159,7 @@ impl Gpu { variant: shared.variant, queues <- runtime::Registry::new( shared.clusters, + shared.deflake, shared.descriptor_flags, shared.variant, ), diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 66c515f6f59de..b916a24e66c61 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -154,6 +154,7 @@ pub(crate) struct Pair { initial_packet: u64, graph_started: bool, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, installed: [bool; 2], @@ -200,6 +201,7 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, ) -> Result> { @@ -258,6 +260,7 @@ impl Pair { initial_packet: first.id, graph_started: false, clusters, + deflake, descriptor_flags, variant, installed: [false; 2], @@ -361,7 +364,7 @@ impl Pair { let node = work.node_va()?; let word = work.word(); let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; - let deflake_layout = abi::Deflake::new(self.clusters)?; + let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory .target diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index ce77d870de641..89f4473442c14 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,6 +25,7 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, installed: [bool; 2], @@ -91,6 +92,7 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, installed: self.installed, @@ -232,6 +234,7 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, installed: self.installed, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index cc3e4b3044b03..45d37ccc11a53 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -71,6 +71,7 @@ impl Registry { /// registry on the probe stack. No additional allocation is required. pub(super) fn new( clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, ) -> impl Init { @@ -83,7 +84,7 @@ impl Registry { metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, - render: render::RenderState::new(clusters, descriptor_flags)?, + render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -411,10 +412,15 @@ pub(super) enum Rebind { #[kunit_tests(asahi_g17_uninstalled_cleanup)] mod uninstalled_cleanup_tests { use super::*; + use crate::g17::fw::render::Deflake; use crate::g17::fw::variant::V26A428; #[test] fn empty_reservation_prevents_reuse_until_explicit_cancellation() { - let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); let first = registry.reserve_compute(1).unwrap(); assert!(registry.reserved(first).is_ok()); assert!(registry.reserve_compute(1).is_err()); @@ -1652,11 +1658,16 @@ impl Backend { #[kunit_tests(asahi_g17_shared_capacity)] mod shared_capacity_tests { use super::*; + use crate::g17::fw::render::Deflake; use crate::g17::fw::variant::V26A428; #[test] fn all_logical_render_owners_fit_beside_shared_compute() { - let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); // Physical compute channels are device-owned and bounded independently // of how many logical clients have queued unsignalled work. @@ -1679,7 +1690,11 @@ mod shared_capacity_tests { use crate::g17::fw::variant::V25G83; let limit = super::super::kick::queue_limit(V25G83.qid_floor); assert_eq!(limit, 42); - let mut registry = KBox::init(Registry::new(2, [0; 2], &V25G83), GFP_KERNEL).unwrap(); + let mut registry = KBox::init( + Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), + GFP_KERNEL, + ) + .unwrap(); for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { registry.reserve_compute(owner).unwrap(); } @@ -1697,7 +1712,11 @@ mod shared_capacity_tests { #[test] fn four_clients_have_capacity_before_any_owner_closes() { - let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); for client in 0..4u64 { for queue in 0..9u64 { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a399a1532550f..5bb1943cd85df 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -24,6 +24,7 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -77,7 +78,11 @@ pub(super) struct RenderState { } impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + pub(super) fn new( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + ) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -88,6 +93,7 @@ impl RenderState { Ok(Self { config: Config { clusters, + deflake, descriptor_flags, }, entries, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 18c6abf294fc4..e2034c3176595 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -274,6 +274,7 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, + resources.config.deflake, resources.config.descriptor_flags, self.shared.variant, ) diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index db0c33fff8485..7a4d6f288dc3d 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -9,7 +9,7 @@ use crate::g17::fw::{ self, initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, }; -use crate::g17::{Config, CrashBuffers, FixedBuffer, Identity}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; use crate::mmu::UatQuirks; @@ -64,6 +64,12 @@ pub(crate) const CONFIG: Config = Config { name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", variant: &fw::variant::V25G83, }]), + // Measured on the 20-core M4 Pro; other core counts are refused at probe. + core_layout: CoreLayout { + cores: Some((2, 20)), + operand_state: fw::compute::OperandState::TWO_BY_TEN, + deflake: fw::render::Deflake::TWO_CLUSTERS, + }, }; /// Chip identifier written to the hardware data. diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index f5ed7bc80963b..75925713bf10b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -54,6 +54,12 @@ pub(crate) const CONFIG: Config = Config { adopt_root_entries: true, fabric_bridge: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + // The layouts the A18 Pro has always run with. + core_layout: crate::g17::CoreLayout { + cores: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; From c0b9fbfa9fb048d5bc67ffb9e17168b686b1962d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:32 +1100 Subject: [PATCH 21/28] drm/asahi: g17: check the voltage cells of the GPU operating points The performance tables take each operating point's voltages from one cell for the whole GPU or one cell per cluster, but nothing checked that a table with several cells has one per cluster. A table with another count filled the lanes of clusters the GPU does not have, or left some of its clusters at zero volts. Refuse the GPU's operating points unless they have one cell or as many cells as the GPU reports clusters. The A18 Pro's table has one cell and is accepted as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/initdata.rs | 20 ++++++++++++++++++-- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 2 +- drivers/gpu/drm/asahi/g17/mod.rs | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6057e7d0ac957..10af261aeb3ac 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -185,6 +185,8 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, + /// GPU clusters, as the GPU reports them. + clusters: u32, } impl BuildEnv<'_> { @@ -714,6 +716,7 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + clusters: u32, active_cores: u32, variant: &super::fw::Variant, ) -> Result { @@ -727,6 +730,7 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + clusters, }; if variant.graph == super::fw::variant::InitGraph::V25G83 { return self.build_25g83(&env, cfg, metrics, active_cores); @@ -1025,7 +1029,7 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev, cfg.opps)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: cfg.windows, @@ -1217,11 +1221,23 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice, count: usize) -> Result> { + /// The GPU's operating points: `count` of them, each with one voltage cell for the whole + /// GPU or one per cluster. + fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; if opps.len() != count { return Err(EINVAL); } + let cells = opps.first().map_or(0, |opp| opp.cells); + if cells != 1 && u32::try_from(cells) != Ok(clusters) { + dev_err!( + dev.as_ref(), + "GPU operating points have {} voltage cells, expected 1 or {}\n", + cells, + clusters + ); + return Err(EINVAL); + } Ok(opps) } diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index bfb7cf1af3ecd..85aa6468fa311 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -98,7 +98,7 @@ impl InitData { *va = self.object(index)?.gpu_va(); } - let opps = Self::opps(env.dev, cfg.opps)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; let hw_args = abi::HwDataArgs { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3906e8c19ff26..527aca52c8c43 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1190,6 +1190,7 @@ impl Gpu { &session.uat, &session._render_global, &session._pm_metrics, + shared.clusters, shared.active_cores, shared.variant, )?; From 1c5a2ef5783a565e6b27da871d30e4e637e1ce69 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:40 +1100 Subject: [PATCH 22/28] drm/asahi: g17: accept only the two host words of a pointer block PointerBlock::word() took any offset other than 0x50 as the word at 0x40, so a pointer layout naming another offset would silently move the producer or the recovery target to the wrong word. Accept only the two host words and fail otherwise, and check the layouts of both firmware variants at build time, so that a wrong layout cannot be built at all. The words written for the existing layouts are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/fw/queue.rs | 46 ++++++++++++++----- drivers/gpu/drm/asahi/g17/fw/variant.rs | 5 ++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 2 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 2 +- 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 5d9a6202943f3..73dfa9862c5b8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -160,8 +160,30 @@ static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); unsafe impl Zeroable for PointerBlock {} impl PointerBlock { + /// Offsets of the two host words a [`PointerLayout`] may name. + const WORD_40: usize = core::mem::offset_of!(Self, word_40); + const WORD_50: usize = core::mem::offset_of!(Self, word_50); + + /// Whether `offset` is one of the host words a [`PointerLayout`] may name. + const fn is_host_word(offset: usize) -> bool { + offset == Self::WORD_40 || offset == Self::WORD_50 + } + + /// Whether `layout` names host words of the block, and two different ones when it has a + /// recovery target. Every firmware variant's layout is checked at build time. + pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { + match layout.recovery_target { + None => Self::is_host_word(layout.producer), + Some(target) => { + Self::is_host_word(layout.producer) + && Self::is_host_word(target) + && target != layout.producer + } + } + } + /// Returns the cursors of an empty item ring. - pub(crate) fn new(layout: &PointerLayout) -> Self { + pub(crate) fn new(layout: &PointerLayout) -> Result { let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], @@ -175,17 +197,17 @@ impl PointerBlock { unk_64: [0; 7], }; if let Some(target) = layout.recovery_target { - block.word(target).store(u32::MAX, Ordering::Relaxed); + block.word(target)?.store(u32::MAX, Ordering::Relaxed); } - block + Ok(block) } - /// The host word at `offset`, one of the two [`PointerLayout`] names. - fn word(&self, offset: usize) -> &AtomicU32 { - if offset == core::mem::offset_of!(Self, word_50) { - &self.word_50 - } else { - &self.word_40 + /// The host word at `offset`, which must be one a [`PointerLayout`] may name. + fn word(&self, offset: usize) -> Result<&AtomicU32> { + match offset { + Self::WORD_40 => Ok(&self.word_40), + Self::WORD_50 => Ok(&self.word_50), + _ => Err(EINVAL), } } @@ -196,10 +218,12 @@ impl PointerBlock { /// queues. Only valid while the firmware does not run the queue, with all /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { + let producer = self.word(layout.producer)?; self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.word(layout.producer).store(base, Ordering::Relaxed); + producer.store(base, Ordering::Relaxed); + Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index bb30925f54160..2f7b598733368 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -126,3 +126,8 @@ pub(crate) const V25G83: Variant = Variant { lead_commands: true, programs: super::render::RegisterPrograms::V25G83, }; + +const _: () = { + assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); +}; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index bf8d7ffa6afb4..4410fc79f8017 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -155,7 +155,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(&pointers); + view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 09ccb689b6632..94b7f42ef3fff 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -160,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(&pointers); + *view = queue::PointerBlock::new(&pointers)?; Ok(()) })?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index f718cd47f90cf..da90488bfedec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -40,7 +40,7 @@ impl Pair { for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(&layout, 0); + unsafe { &*block }.rebase(&layout, 0)?; } fence(Ordering::SeqCst); self.cursors = [0; 2]; From df6b70b85cbfada36604f511b9ba8da21b13eb7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:40 +1100 Subject: [PATCH 23/28] drm/asahi: g17: pass the fragment ID to the barrier unchanged write_leads() narrowed the fragment descriptor's 64-bit object ID to 32 bits before FragmentBarrier::new() widened it again into the barrier's 64-bit field, which would truncate any ID above 32 bits. Pass the ID through as it is. The IDs come from 32-bit counters today, so the bytes written do not change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/fw/queue.rs | 4 ++-- drivers/gpu/drm/asahi/g17/queue/render/memory.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 73dfa9862c5b8..a28f98fb88067 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -609,7 +609,7 @@ impl FragmentBarrier { tiling_kick: KickTimestamp, tiling_qid: u32, stamp_value: u32, - fragment_id: u32, + fragment_id: u64, ) -> Self { Self { tag: CommandTag::FragmentBarrier as u32, @@ -620,7 +620,7 @@ impl FragmentBarrier { unk_1c: [0; 8], event: tiling_qid, stamp_self: stamp_value, - fragment_id: fragment_id.into(), + fragment_id, unk_34: 1, unk_38: [0; 0x10], } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 94b7f42ef3fff..80ee0b871cc82 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -659,7 +659,7 @@ impl Memory { tiling_kick, ta.qid, ta.stamp_value, - fragment_id as u32, + fragment_id, ); self.graph.config.write(CONFIG[1] + slot, barrier)?; if init_manager { From d14b566cc309cf6840199c6394dc8f5f75f82f54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:41 +1100 Subject: [PATCH 24/28] drm/asahi: g17: describe the 25G83 barrier and parameter block plainly Two comments named internals of Apple's userspace GPU driver to explain where the 25G83 fragment barrier and the tiler parameter block sizes come from. State what the code does instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/buffer/target.rs | 4 ++-- drivers/gpu/drm/asahi/g17/fw/queue.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bfa64c0b27a6a..64c88da8eca14 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -25,8 +25,8 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } -/// The 25G83 tiler parameter block (the 25G83 Metal driver's default parameter-buffer sizes): -/// the TPC and seven sub-buffers the tiling registers name. Sizes are +/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at +/// the default parameter-buffer sizes. Sizes are /// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index a28f98fb88067..5708a2251a062 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -570,8 +570,7 @@ impl QueueConfig { } /// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work -/// waits for the paired tiling stamp, in addition to the kick-entry dependency, as the 25G83 -/// Metal driver writes it ("emitBarrierCommands"). +/// waits for the paired tiling stamp, in addition to the kick-entry dependency. #[repr(C, packed)] #[derive(Copy, Clone, Debug)] pub(crate) struct FragmentBarrier { From f0f00dca4b09c9caf6f379928827e0d10451c66b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:41 +1100 Subject: [PATCH 25/28] drm/asahi: mem: document that page-by-page invalidation is global PAGE_TLBI is a global static that Uat::new() sets from the SoC's UAT quirks, because the TLB invalidation helpers take no UAT. Say so, and say that it assumes one GPU per system, as every supported system has. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/mem.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 91fd5092aaefc..5f0ab2b4aea4f 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -20,12 +20,16 @@ use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; /// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +/// +/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per +/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second +/// UAT with another policy would change it for both. static PAGE_TLBI: AtomicBool = AtomicBool::new(false); /// Ranges above this many pages are invalidated by ASID in page mode. const PAGE_TLBI_MAX_PAGES: usize = 2048; -/// Selects page-by-page range invalidation for this GPU. +/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). pub(crate) fn set_page_tlbi(enabled: bool) { PAGE_TLBI.store(enabled, Ordering::Relaxed); } From 0c55425a8157326aa0bafd20a1c30a0f34af8650 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:41 +1100 Subject: [PATCH 26/28] dt-bindings: gpu: apple,agx: Require three cells in apple,firmware-version apple,firmware-version is a macOS version (major, minor, patch), and the G17 driver only accepts it with exactly three cells. The binding only limited it to at most three. Require three. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/gpu/apple,agx.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 843d7c4af8f9a..d29b1ab0310aa 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -78,6 +78,7 @@ properties: apple,firmware-version: $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 maxItems: 3 description: macOS version (major, minor, patch) whose boot firmware loaded the GPU From d36196335065346ef57a323e483002151670320c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 12:44:47 +1100 Subject: [PATCH 27/28] arm64: dts: apple: t6040-j616s: Keep the GPU disabled The J616s GPU node carries the operating points and reserved regions of one unit, and the GPU firmware's per-unit values are not filled in from the ADT yet. Another unit needs 5 to 15 mV more in most states, and other units place the regions elsewhere. Keep the GPU disabled until a boot loader fills these in. The GPU's companions follow it. gfx_mbox stays disabled, and the board no longer claims pmp_gfx always-on, because telling the PMP the GPU is on with no GPU driver crashes the PMP firmware right after it starts. ps_afr and ps_gfx are no longer always-on in the SoC file either, so a disabled GPU's power states are not forced on. A board that enables the GPU makes them always on again, and the operating-point tables and reserved regions stay in place for that. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 18 ++++++++---------- arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 7 ++++--- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 76bc9e1adb406..7c563856d136e 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -807,10 +807,13 @@ status = "okay"; }; -&gfx_mbox { - status = "okay"; -}; - +/* + * The GPU stays disabled until a boot loader fills in this unit's operating + * points, leakage and regions from the ADT; the values here are one unit's. + * Enabling it also takes gfx_mbox, keeps ps_afr and ps_gfx always on and + * claims pmp_gfx always-on, so that the PMP sees the GPU on exactly while it + * is powered. + */ &gpu { memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_fw_data>, <&uat_l2_pagetables>; @@ -819,12 +822,7 @@ operating-points-v2 = <&gpu_opp>; apple,cs-opp = <&gpu_cs_opp>; apple,afr-opp = <&gpu_afr_opp>; - status = "okay"; -}; - -/* The GPU firmware runs from boot on (ps_gfx is always on): the PMP must see the GPU on. */ -&pmp_gfx { - apple,always-on; + status = "disabled"; }; /* diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index 2d3ae512144bc..8c05fb4d4d5d3 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -14,10 +14,12 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "afr"; - apple,always-on; }; - /* The GPU firmware keeps running and does not survive this domain going off. */ + /* + * The GPU firmware keeps running and does not survive this domain going + * off, so a board that enables the GPU keeps it and ps_afr always on. + */ ps_gfx: power-controller@110 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x110 4>; @@ -25,7 +27,6 @@ #reset-cells = <0>; label = "gfx"; power-domains = <&ps_afr>; - apple,always-on; }; }; From b9a8f3938f3daf5926b7c86204113f6730aec136 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 16:59:36 +1100 Subject: [PATCH 28/28] drm/asahi: g17: pass the queue count to Pool::new() in the fault admission test This series makes Pool::new() take the number of queues, and #413 added a KUnit case that still calls it without one. Pass MAX_EXECUTION_CONTEXTS like the other admission tests. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 94aed217e421e..ebdc79d2b2dd3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -781,7 +781,7 @@ mod fault_capacity_tests { #[test] fn a_guilty_failure_makes_a_spared_quarantine_guilty() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot();