diff --git a/MAINTAINERS b/MAINTAINERS
index b13c1507cf391..18d06932e001b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2794,9 +2794,11 @@ F:	tools/asahi/apple-pwm-pm-tests.py
 F:	drivers/reset/reset-apple-cio.c
 F:	drivers/rtc/rtc-macsmc.c
 F:	drivers/soc/apple/*
+F:	tools/asahi/m3-uapi-*
 F:	drivers/soc/apple/clpc.kunitconfig
 F:	tools/asahi/pmp-bootargs-tests.rs
 F:	drivers/spi/spi-apple-test.c
+F:	tools/asahi/test-m3-uapi.py
 F:	drivers/spi/spi-apple.c
 F:	drivers/spmi/spmi-apple-controller.c
 F:	drivers/thermal/apple-pmp*
diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config
index 126f60eee6c74..5581318240726 100644
--- a/arch/arm64/configs/asahi.config
+++ b/arch/arm64/configs/asahi.config
@@ -65,6 +65,7 @@ CONFIG_DRM=y
 CONFIG_DRM_ASAHI=m
 CONFIG_DRM_ASAHI_G15=y
 CONFIG_DRM_ASAHI_G16=y
+CONFIG_DRM_ASAHI_M3_T8122=y
 CONFIG_DRM_ADP=m
 CONFIG_DRM_APPLE=m
 CONFIG_DRM_APPLE_DCP_T8132=y
diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig
index 3838ee981e85f..d383802ee8825 100644
--- a/drivers/gpu/drm/asahi/Kconfig
+++ b/drivers/gpu/drm/asahi/Kconfig
@@ -79,6 +79,22 @@ config DRM_ASAHI_DEBUG_ALLOCATOR
 	  Say N unless you are debugging firmware structures or porting to a
 	  new firmware version.
 
+config DRM_ASAHI_M3_T8122
+	bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)"
+	depends on DRM_ASAHI
+	# The runtime adopts the firmware's UAT through the G15 backend's handoff helpers.
+	depends on DRM_ASAHI_G15
+	select CRYPTO_LIB_SHA256
+	default n
+	help
+	  Build the M3 runtime that starts the GPU of the M3 MacBook Air
+	  (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves
+	  loaded. On the listed Air boards it starts after the boot loader
+	  handed the GPU over and resource/firmware identity checks pass.
+	  asahi.t8122_start=0 keeps startup disabled for recovery. Unknown
+	  boards and firmware do not gain automatic admission. Other GPUs
+	  are not affected.
+
 config DRM_ASAHI_G15
 	bool "Apple M3 Pro/Max (G15) support"
 	depends on DRM_ASAHI
diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs
index 479d9763cc7ed..6161d8d8dd748 100644
--- a/drivers/gpu/drm/asahi/debug.rs
+++ b/drivers/gpu/drm/asahi/debug.rs
@@ -54,6 +54,13 @@ pub(crate) enum DebugFlags {
     DeviceControlCh = 21,
     FwCtlCh = 22,
 
+    // Host submission/completion timestamps without verbose firmware tracing.
+    SubmitTiming = 23,
+    // Low-volume M3 command/geometry totals without per-IRQ printk.
+    M3SubmitSummary = 24,
+    // Per-slot retired timestamps, without enabling firmware ktrace.
+    M3PassTiming = 25,
+
     // 32-35: Allocator debugging
     FillAllocations = 32,
     DebugAllocations = 33,
diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs
index f3707ec05066a..f07596f9e6b16 100644
--- a/drivers/gpu/drm/asahi/driver.rs
+++ b/drivers/gpu/drm/asahi/driver.rs
@@ -52,6 +52,7 @@ pub(crate) struct AsahiPlatformDriverData<'bound> {
     g17: Option<Arc<g17::Gpu>>,
     #[cfg(CONFIG_DRM_ASAHI_G16)]
     pub(crate) g16: Option<Arc<crate::g16::Gpu>>,
+    m3: Option<M3Backend>,
     /// Raw `struct device *` of the platform device. Stored once at probe
     /// time so the `Drop` impl can find the sysfs file on unregister.
     raw_dev: *mut bindings::device,
@@ -65,6 +66,11 @@ unsafe impl Send for AsahiPlatformDriverData<'_> {}
 // SAFETY: As above.
 unsafe impl Sync for AsahiPlatformDriverData<'_> {}
 
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+type M3Backend = Arc<crate::m3::m3_drm::Backend>;
+#[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))]
+type M3Backend = core::convert::Infallible;
+
 /// Holds a reference to the top-level GPU object.
 #[pin_data]
 pub(crate) struct AsahiDrmRegistrationData<'drm> {
@@ -245,6 +251,10 @@ impl platform::Driver for AsahiPlatformDriver {
             // Driver data is dropped after devres, so CPU control must happen in unbind.
             gpu.shutdown();
         }
+        #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+        if let Some(m3) = this.m3.as_ref() {
+            m3.stop();
+        }
         this.close_binding();
     }
 
@@ -264,6 +274,8 @@ impl platform::Driver for AsahiPlatformDriver {
         let cfg = match info.ok_or(ENODEV)? {
             ProbeConfig::Legacy(cfg) => *cfg,
             ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg),
+            #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+            ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev),
             ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)),
             #[cfg(CONFIG_DRM_ASAHI_M5)]
             ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev),
@@ -351,6 +363,7 @@ impl platform::Driver for AsahiPlatformDriver {
             _device: reg.device().into(),
             _reg: reg,
             g17: None,
+            m3: None,
             #[cfg(CONFIG_DRM_ASAHI_G16)]
             g16: None,
             raw_dev,
@@ -388,6 +401,7 @@ impl AsahiPlatformDriver {
             _device: reg.device().into(),
             _reg: reg,
             g17: None,
+            m3: None,
             #[cfg(CONFIG_DRM_ASAHI_G16)]
             g16: None,
             raw_dev: core::ptr::null_mut(),
@@ -396,6 +410,46 @@ impl AsahiPlatformDriver {
         })
     }
 
+    /// The M3 MacBook Air (T8122): the M3 runtime, started only when its identity gate, the
+    /// backend selection and the start experiment (`asahi.t8122_start=1`) admit it; otherwise
+    /// the probe is refused as without the runtime.
+    #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+    fn probe_t8122<'bound>(
+        pdev: &'bound platform::Device<Core<'_>>,
+    ) -> Result<AsahiPlatformDriverData<'bound>> {
+        let soc = crate::m3::t8122_admitted(pdev)?;
+        // SAFETY: The admitted M3 GPU's UAT outputs 42-bit physical addresses.
+        unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? };
+        let resources = regs::Resources::new(pdev)?;
+        let drm = drm::UnregisteredDevice::<AsahiDrmDriver>::new(pdev, Ok(()))?;
+        let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? };
+        let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?;
+        let gpu = backend.clone() as Arc<dyn gpu::Gpu>;
+        let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources });
+        // SAFETY: The registration is stored and unregistered by platform data destruction,
+        // which drops the runtime with it.
+        let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } {
+            Ok(reg) => reg,
+            Err(error) => {
+                backend.stop();
+                return Err(error);
+            }
+        };
+        dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n");
+        Ok(AsahiPlatformDriverData {
+            binding,
+            #[cfg(CONFIG_DRM_ASAHI_G16)]
+            g16: None,
+            _device: reg.device().into(),
+            _reg: reg,
+            g17: None,
+            m3: Some(backend),
+            raw_dev: core::ptr::null_mut(),
+            raw_stats: core::ptr::null_mut(),
+            stats: None,
+        })
+    }
+
     fn probe_g17<'bound>(
         pdev: &'bound platform::Device<Core<'_>>,
         cfg: &'static g17::Config,
@@ -451,6 +505,7 @@ impl AsahiPlatformDriver {
             _device: reg.device().into(),
             _reg: reg,
             g17: Some(gpu),
+            m3: None,
             #[cfg(CONFIG_DRM_ASAHI_G16)]
             g16: None,
             raw_dev,
@@ -486,6 +541,7 @@ impl AsahiPlatformDriver {
             _device: reg.device().into(),
             _reg: reg,
             g17: None,
+            m3: None,
             #[cfg(CONFIG_DRM_ASAHI_G16)]
             g16: None,
             raw_dev: core::ptr::null_mut(),
@@ -524,6 +580,7 @@ impl AsahiPlatformDriver {
             _device: reg.device().into(),
             _reg: reg,
             g17: None,
+            m3: None,
             g16: Some(gpu),
             raw_dev,
             raw_stats,
@@ -579,6 +636,7 @@ impl AsahiPlatformDriver {
                 _device: reg.device().into(),
                 _reg: reg,
                 g17: None,
+                m3: None,
                 #[cfg(CONFIG_DRM_ASAHI_G16)]
                 g16: None,
                 raw_dev: core::ptr::null_mut(),
@@ -608,5 +666,9 @@ impl Drop for AsahiPlatformDriverData<'_> {
             // Also covers failure to allocate platform driver data after the probe body returned.
             gpu.shutdown();
         }
+        #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+        if let Some(m3) = self.m3.as_ref() {
+            m3.stop();
+        }
     }
 }
diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs
index 3778ee5a4aa68..ffe5f0e846930 100644
--- a/drivers/gpu/drm/asahi/file.rs
+++ b/drivers/gpu/drm/asahi/file.rs
@@ -282,6 +282,9 @@ pub(crate) struct File {
     mem: Arc<memacct::Client>,
     /// Completed engine time, retained by queues after handle destruction.
     usage: Arc<crate::usage::Client>,
+    /// Largest size this file has asked GET_PARAMS for the global parameters with
+    /// ([`gpu::Gpu::admit_vm`]).
+    params_size: core::sync::atomic::AtomicU64,
     /// The queues were drained once for an exiting task.
     drained: AtomicBool,
 }
@@ -376,6 +379,7 @@ impl File {
                 (*slot).user_range = user_range;
                 addr_of_mut!((*slot).mem).write(mem);
                 addr_of_mut!((*slot).usage).write(usage);
+                addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0));
                 addr_of_mut!((*slot).drained).write(AtomicBool::new(false));
                 Ok(())
             })
@@ -480,6 +484,9 @@ impl File {
             core::slice::from_raw_parts(core::ptr::from_ref(&params).cast::<u8>(), size)
         })?;
 
+        file.inner()
+            .params_size
+            .fetch_max(data.size, core::sync::atomic::Ordering::Relaxed);
         Ok(0)
     }
 
@@ -551,6 +558,13 @@ impl File {
             // Every other runtime here reports DRM_ASAHI_GPU_HAL_LEGACY.
             return Err(EINVAL);
         }
+        reg_data.gpu.admit_vm(
+            device.as_ref().as_ref(),
+            file.inner()
+                .params_size
+                .load(core::sync::atomic::Ordering::Relaxed),
+            data.flags,
+        )?;
 
         let kernel_range = data.kernel_start..data.kernel_end;
         let user_range = file.inner().user_range.clone();
@@ -647,7 +661,7 @@ impl File {
         }
         let dummy_mapping = dummy_obj.map_at(
             &vm,
-            gpu.uat_geometry().unk_page(),
+            gpu.unknown_page(),
             mmu::PROT_GPU_SHARED_RW,
             true,
         )?;
@@ -1378,9 +1392,15 @@ impl File {
             cls_pr_debug!(Errors, "queue_create: Invalid priority\n");
             return Err(EINVAL);
         }
-        // HIGH has only been run on G17; G13 and G14 keep refusing it.
+        // Keep support tied to a backend and admitted profile. G13/G14 and
+        // unqualified M3 profiles retain their previous rejection.
+        let high_supported = reg_data.gpu.as_any().is::<crate::g17::Gpu>();
+        #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+        let high_supported = high_supported || reg_data.gpu.as_any()
+            .downcast_ref::<crate::m3::m3_drm::Backend>()
+            .is_some_and(|gpu| gpu.supports_high_priority());
         if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH
-            && !reg_data.gpu.as_any().is::<crate::g17::Gpu>()
+            && !high_supported
         {
             cls_pr_debug!(
                 Errors,
diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs
index 52261d10757d1..782cd47fc7111 100644
--- a/drivers/gpu/drm/asahi/float.rs
+++ b/drivers/gpu/drm/asahi/float.rs
@@ -44,16 +44,16 @@ struct F32U {
 }
 
 impl F32 {
-    /// Return the IEEE-754 binary32 representation.
-    pub(crate) const fn to_bits(self) -> u32 {
-        self.0
-    }
-
     /// Convert a raw 32-bit representation into an F32
     pub(crate) const fn from_bits(u: u32) -> F32 {
         F32(u)
     }
 
+    /// Convert to the raw IEEE 754 bits.
+    pub(crate) const fn to_bits(self) -> u32 {
+        self.0
+    }
+
     // Convert a `f32` value into an F32
     //
     // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely.
diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs
index 9ffd858598b1c..ae0512de0b156 100644
--- a/drivers/gpu/drm/asahi/g15/mod.rs
+++ b/drivers/gpu/drm/asahi/g15/mod.rs
@@ -69,10 +69,10 @@ mod pools;
 mod power;
 mod queue;
 mod recovery;
-mod regs;
+pub(crate) mod regs;
 mod slots;
 mod timestamp;
-mod uat;
+pub(crate) mod uat;
 mod vm;
 mod wire;
 
diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs
index d5435e9f1c7ea..05a1b390b3dff 100644
--- a/drivers/gpu/drm/asahi/g15/regs.rs
+++ b/drivers/gpu/drm/asahi/g15/regs.rs
@@ -152,6 +152,21 @@ impl Mapping {
     pub(crate) fn phys(&self) -> u64 {
         self.phys
     }
+
+    /// Cleans the whole mapping to the point of coherency when it is the kernel's cacheable
+    /// mapping of reserved System RAM ([`Self::region`]); device and write-combined mappings
+    /// need nothing.
+    pub(crate) fn clean_poc(&self) {
+        if self.kind != Kind::WriteBack {
+            return;
+        }
+        let start = self.raw.addr();
+        // SAFETY: `start..start + maxsize` is this live mapping; cleaning a cacheable range has
+        // no other effect.
+        unsafe {
+            bindings::dcache_clean_poc(start as _, (start + self.raw.size()) as _);
+        }
+    }
 }
 
 impl Drop for Mapping {
diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs
index 53e99f5a772b2..0f91f7c9f0610 100644
--- a/drivers/gpu/drm/asahi/g15/uat.rs
+++ b/drivers/gpu/drm/asahi/g15/uat.rs
@@ -202,16 +202,41 @@ pub(crate) struct Window {
 impl Window {
     /// Maps the UAT handoff regions ([`Mapping::region`]).
     pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result<Self> {
+        Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start)
+    }
+
+    /// Maps the UAT handoff regions at the given physical addresses: the handoff page, the
+    /// TTBAT, the firmware kernel root and the shared L2 page. The M3 T8122 runtime takes them
+    /// from its admitted reserved-memory regions.
+    pub(crate) fn from_regions(handoff: u64, ttbs: u64, root: u64, shared_l2: u64) -> Result<Self> {
         Ok(Self {
-            handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?,
-            ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?,
-            root: Mapping::region(adt.tables.start, hw::PAGE as usize)?,
-            shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?,
+            handoff: Mapping::region(handoff, hw::HANDOFF_SIZE as usize)?,
+            ttbs: Mapping::region(ttbs, hw::TTBS_SIZE as usize)?,
+            root: Mapping::region(root, hw::PAGE as usize)?,
+            shared_l2: Mapping::region(shared_l2, hw::SHARED_L2_SIZE as usize)?,
             image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?,
             snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?,
         })
     }
 
+    /// [`Self::sync_range`] over the whole host window, then [`Self::clean_tables`].
+    pub(crate) fn sync_all(&mut self, uat: &mmu::Uat) -> Result {
+        self.sync_inner(uat, None)?;
+        self.clean_tables();
+        Ok(())
+    }
+
+    /// Cleans the firmware-walked pages this window writes (the kernel root, the shared L2,
+    /// the TTBAT and the handoff page) to the point of coherency. Only a region that is
+    /// reserved System RAM, mapped cacheable, has anything to clean: the UAT walker does not
+    /// snoop the CPU caches. The M3 T8122 runtime calls it after [`Self::bootstrap`].
+    pub(crate) fn clean_tables(&self) {
+        for map in [&self.root, &self.shared_l2, &self.ttbs, &self.handoff] {
+            map.clean_poc();
+        }
+        mb();
+    }
+
     /// Cold-start check: all 128 TTBAT words are zero.
     pub(crate) fn ttbat_is_clear(&self) -> Result<bool> {
         let t = self.ttbs.io();
diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs
index 0a7a3e6ce065b..c3151f82c270b 100644
--- a/drivers/gpu/drm/asahi/gpu.rs
+++ b/drivers/gpu/drm/asahi/gpu.rs
@@ -361,6 +361,14 @@ pub(crate) trait Gpu: Send + Sync {
     fn base_clock_hz(&self) -> u32;
     /// Return the GPU address space geometry.
     fn uat_geometry(&self) -> mmu::UatGeometry;
+    /// Admit a client's parameter ABI before creating its VM.
+    fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result {
+        Ok(())
+    }
+    /// GPU VA of each VM's dummy page, which must match the address the firmware was given.
+    fn unknown_page(&self) -> u64 {
+        self.uat_geometry().unk_page()
+    }
     /// Create a new `Vm` with the given range reserved for kernel-managed objects.
     fn new_vm(&self, kernel_range: Range<u64>) -> Result<mmu::Vm>;
     /// Create a VM charged to a DRM file when the backend bounds retained bindings.
diff --git a/drivers/gpu/drm/asahi/m3/agx_attachments.rs b/drivers/gpu/drm/asahi/m3/agx_attachments.rs
new file mode 100644
index 0000000000000..5aeb3c7335c22
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_attachments.rs
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+pub(crate) const SIZE: usize = 0x104;
+pub(crate) const CAPACITY: usize = 16;
+
+#[derive(Clone, Copy)]
+pub(crate) struct Attachments(pub(crate) [u8; SIZE]);
+
+impl Attachments {
+    pub(crate) const EMPTY: Self = Self([0; SIZE]);
+    pub(crate) fn count(&self) -> u8 { self.0[0x100] }
+
+    /// Normalize byte ranges to the firmware's 128-byte cache-line units.
+    /// The caller must validate the entire normalized writable GPU range.
+    pub(crate) fn range(address: u64, size: u64) -> Result<(u64, u64), ()> {
+        if address == 0 || size == 0 { return Err(()); }
+        let end = address.checked_add(size).and_then(|v| v.checked_add(127)).ok_or(())? & !127;
+        let start = address & !127;
+        if start == 0 || end > 1 << 42 || (end - start) >> 7 > u32::MAX as u64 {
+            return Err(());
+        }
+        Ok((start, end - start))
+    }
+
+    pub(crate) fn new(ranges: &[(u64, u64)]) -> Result<Self, ()> {
+        if ranges.len() > CAPACITY { return Err(()); }
+        let mut out = Self::EMPTY;
+        for (record, &(address, size)) in out.0[..0x100].chunks_exact_mut(16).zip(ranges) {
+            let (address, size) = Self::range(address, size)?;
+            record[..8].copy_from_slice(&address.to_le_bytes());
+            record[8..12].copy_from_slice(&((size >> 7) as u32).to_le_bytes());
+            record[12..14].copy_from_slice(&0x18u16.to_le_bytes());
+            record[14..16].copy_from_slice(&1u16.to_le_bytes());
+        }
+        out.0[0x100..].copy_from_slice(&(ranges.len() as u32).to_le_bytes());
+        Ok(out)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn covers_both_partial_edge_lines() {
+        let a = Attachments::new(&[(0x100007f, 2)]).unwrap();
+        assert_eq!(u64::from_le_bytes(a.0[..8].try_into().unwrap()), 0x1000000);
+        assert_eq!(u32::from_le_bytes(a.0[8..12].try_into().unwrap()), 2);
+        assert_eq!(&a.0[12..16], &[0x18, 0, 1, 0]);
+        assert_eq!(a.count(), 1);
+        assert!(a.0[16..0x100].iter().all(|v| *v == 0));
+    }
+    #[test]
+    fn rejects_unrepresentable_ranges_and_count() {
+        for range in [(0, 128), (128, 0), (u64::MAX - 1, 8),
+                      ((1 << 42) - 128, 129), (128, (u32::MAX as u64 + 1) << 7)] {
+            assert!(Attachments::new(&[range]).is_err());
+        }
+        assert!(Attachments::new(&[(128, 128); CAPACITY + 1]).is_err());
+        let full = Attachments::new(&[(128, 128); CAPACITY]).unwrap();
+        assert_eq!(full.count(), 16);
+        assert_eq!(&full.0[0x100..], &[16, 0, 0, 0]);
+        assert_eq!(Attachments::new(&[]).unwrap().0, Attachments::EMPTY.0);
+    }
+    #[test]
+    fn matches_native_m4_writable_buffer_descriptor() {
+        let encoded = Attachments::new(&[(0x100000a8000, 0x400000)]).unwrap();
+        assert_eq!(&encoded.0[..16], &[
+            0, 0x80, 0x0a, 0, 0, 1, 0, 0, 0, 0x80, 0, 0, 0x18, 0, 1, 0]);
+        assert_eq!(encoded.count(), 1);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/agx_compute.rs b/drivers/gpu/drm/asahi/m3/agx_compute.rs
new file mode 100644
index 0000000000000..d58005c504f7a
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_compute.rs
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+/// Complete client CDM range and its USC address window.
+#[derive(Clone, Copy)]
+pub(crate) struct Control {
+    pub base: u64,
+    pub end: u64,
+    pub usc_base: u64,
+}
diff --git a/drivers/gpu/drm/asahi/m3/agx_memory.rs b/drivers/gpu/drm/asahi/m3/agx_memory.rs
new file mode 100644
index 0000000000000..521990b4aad21
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_memory.rs
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+// Firmware validates Timestamp user destinations against a dedicated 64 MiB
+// interval published in HwDataB+0x28. Reserve the last 64 MiB of the
+// firmware VM selector, disjoint from MMIO/command allocations above.
+pub(crate) const TIMESTAMP_RANGE: core::ops::Range<u64> =
+    0xffff_fc2f_fc00_0000..0xffff_fc30_0000_0000;
+
+pub(crate) fn publish() {
+    // SAFETY: Orders preceding WC buffer stores before a device doorbell.
+    unsafe { core::arch::asm!("dsb oshst", options(nostack, preserves_flags)) };
+}
diff --git a/drivers/gpu/drm/asahi/m3/agx_resources.rs b/drivers/gpu/drm/asahi/m3/agx_resources.rs
new file mode 100644
index 0000000000000..8fdf87bdb046d
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_resources.rs
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! J713 bootloader resource handoff. Reading this metadata does not map memory,
+//! claim UAT ownership, authenticate firmware, or grant permission to start it.
+//! Firmware ABI admission and an owned runtime are separate steps.
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct Region {
+    pub(crate) base: u64,
+    pub(crate) size: u64,
+}
+
+impl Region {
+    fn end(self) -> Option<u64> {
+        self.base.checked_add(self.size)
+    }
+
+    fn valid(self) -> bool {
+        self.base != 0
+            && self.size != 0
+            && (self.base | self.size) & 0x3fff == 0
+            && self.end().is_some_and(|end| end <= 1 << 42)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum ResourceError {
+    Version,
+    Region,
+    Overlap,
+    FirmwareLayout,
+}
+
+#[derive(Debug)]
+pub(crate) struct Resources {
+    /// ttbs, pagetables, handoff, shared-l2, fw-text, fw-data.
+    pub(crate) regions: [Region; 6],
+    pub(crate) firmware_vas: [u64; 2],
+}
+
+impl Resources {
+    pub(crate) fn validate(
+        version: u32,
+        regions: [Region; 6],
+        firmware_vas: [u64; 2],
+        firmware_flags: [u32; 2],
+    ) -> Result<Self, ResourceError> {
+        if version != 1 {
+            return Err(ResourceError::Version);
+        }
+        for (i, region) in regions.iter().enumerate() {
+            if !region.valid() {
+                return Err(ResourceError::Region);
+            }
+            for other in &regions[..i] {
+                if region.base < other.end().unwrap() && other.base < region.end().unwrap() {
+                    return Err(ResourceError::Overlap);
+                }
+            }
+        }
+        // The inherited 42-bit page-table roots and all 64 16-byte TTBAT
+        // entries fit their reserved regions. No panel-sized allocation pool.
+        if firmware_flags != [1, 0]
+            || firmware_vas[0] != 0xffff_fc00_0000_0000
+            || firmware_vas[0].checked_add(regions[4].size) != Some(firmware_vas[1])
+            || firmware_vas[1].checked_add(regions[5].size).is_none()
+        {
+            return Err(ResourceError::FirmwareLayout);
+        }
+        Ok(Self {
+            regions,
+            firmware_vas,
+        })
+    }
+}
+
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn observed() -> [Region; 6] {
+        [
+            (0x103fffb8000, 0x4000),
+            (0x103fff78000, 0x40000),
+            (0x103fff70000, 0x4000),
+            (0x103fff74000, 0x4000),
+            (0x10000cc0000, 0x60000),
+            (0x10001df4000, 0x13c000),
+        ]
+        .map(|(base, size)| Region { base, size })
+    }
+    const VAS: [u64; 2] = [0xfffffc0000000000, 0xfffffc0000060000];
+
+    #[test]
+    fn observed_and_relocated_boot_addresses() {
+        let res = Resources::validate(1, observed(), VAS, [1, 0]).unwrap();
+        assert_eq!(res.firmware_vas, VAS);
+        assert_eq!(res.regions[2].base, 0x103fff70000);
+        let relocated = observed().map(|r| Region {
+            base: r.base + 0x400000000,
+            ..r
+        });
+        assert!(Resources::validate(1, relocated, VAS, [1, 0]).is_ok());
+    }
+
+    /// Relocating the UAT reservations preserves their sizes and ownership checks.
+    #[test]
+    fn relocated_uat_regions_keep_firmware_segments() {
+        let mut regions = observed();
+        for region in &mut regions[..4] {
+            region.base -= 0x40000;
+        }
+        let res = Resources::validate(1, regions, VAS, [1, 0]).unwrap();
+        assert_eq!(res.regions[0].base, 0x103fff78000);
+        assert_eq!(res.regions[4], observed()[4]);
+        assert_eq!(res.regions[5], observed()[5]);
+    }
+
+    #[test]
+    fn rejects_missing_overlapping_unaligned_and_out_of_range_memory() {
+        for r in [
+            Region { base: 0, size: 0 },
+            Region {
+                base: 0x12345,
+                size: 0x4000,
+            },
+            Region {
+                base: 1 << 42,
+                size: 0x4000,
+            },
+            Region {
+                base: u64::MAX & !0x3fff,
+                size: 0x4000,
+            },
+        ] {
+            let mut bad = observed();
+            bad[0] = r;
+            assert!(matches!(
+                Resources::validate(1, bad, VAS, [1, 0]),
+                Err(ResourceError::Region)
+            ));
+        }
+        let mut bad = observed();
+        bad[0] = bad[1];
+        assert!(matches!(
+            Resources::validate(1, bad, VAS, [1, 0]),
+            Err(ResourceError::Overlap)
+        ));
+    }
+
+    #[test]
+    fn rejects_unknown_handoff_and_segment_layouts() {
+        assert!(matches!(
+            Resources::validate(0, observed(), VAS, [1, 0]),
+            Err(ResourceError::Version)
+        ));
+        for (vas, flags) in [
+            (VAS, [0, 1]),
+            ([VAS[0], VAS[1] + 0x4000], [1, 0]),
+            ([0xfffffe0000000000, VAS[1]], [1, 0]),
+        ] {
+            assert!(matches!(
+                Resources::validate(1, observed(), vas, flags),
+                Err(ResourceError::FirmwareLayout)
+            ));
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/agx_status.rs b/drivers/gpu/drm/asahi/m3/agx_status.rs
new file mode 100644
index 0000000000000..1d83086a1d155
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_status.rs
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! CPU-owned sticky VM failure reporting. No GPU mapping or firmware access.
+use core::sync::atomic::{AtomicI32, Ordering};
+
+pub(crate) const PARAM_GROUP_VM_STATUS: u32 = 1;
+pub(crate) const FEATURE_VM_STATUS: u64 = 1 << 1;
+pub(crate) const VM_STATUS_SIZE: usize = 16;
+
+/// Shared by the VM and accepted jobs; never cleared by queue destruction or
+/// successful work. A newly created VM receives a new status object.
+pub(crate) struct VmStatus { error: AtomicI32 }
+impl VmStatus {
+    pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0) } }
+    pub(crate) fn record(&self, error: i32) {
+        if error < 0 {
+            let _ = self.error.compare_exchange(0, error, Ordering::AcqRel, Ordering::Acquire);
+        }
+    }
+    pub(crate) fn get(&self) -> i32 { self.error.load(Ordering::Acquire) }
+}
+
+/// GET_PARAMS group 1 is an in/out request: vm_id, flags=0, error(out), pad=0.
+/// Decode once, before lookup, so a racing userspace edit cannot change owner.
+pub(crate) fn decode_request(bytes: &[u8; VM_STATUS_SIZE]) -> Option<u32> {
+    let word = |i| u32::from_ne_bytes(bytes[i..i+4].try_into().unwrap());
+    if word(4) != 0 || word(12) != 0 { return None; }
+    Some(word(0))
+}
+pub(crate) fn encode_response(vm_id: u32, error: i32) -> [u8; VM_STATUS_SIZE] {
+    let mut bytes = [0; VM_STATUS_SIZE];
+    bytes[0..4].copy_from_slice(&vm_id.to_ne_bytes());
+    bytes[8..12].copy_from_slice(&error.to_ne_bytes());
+    bytes
+}
diff --git a/drivers/gpu/drm/asahi/m3/agx_uapi.rs b/drivers/gpu/drm/asahi/m3/agx_uapi.rs
new file mode 100644
index 0000000000000..60107f8833edc
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/agx_uapi.rs
@@ -0,0 +1,721 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+#![cfg_attr(not(test), allow(dead_code))]
+
+//! Shared modern-Asahi UAPI parsing.
+//!
+//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`,
+//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates
+//! the queue's 4 GiB USC window and every userspace range before constructing
+//! M3 render parameters or compute-entry operands. This module allocates no
+//! object, publishes no queue state, and performs no firmware or MMIO action.
+
+pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8;
+pub(crate) const UAPI_RENDER_SIZE: usize = 256;
+pub(crate) const UAPI_COMPUTE_SIZE: usize = 64;
+pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24;
+pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16;
+pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64;
+pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff;
+pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32;
+const COMMAND_RENDER: u16 = 0;
+const COMMAND_COMPUTE: u16 = 1;
+const SET_VERTEX_ATTACHMENTS: u16 = 2;
+const SET_FRAGMENT_ATTACHMENTS: u16 = 3;
+const SET_COMPUTE_ATTACHMENTS: u16 = 4;
+
+const RENDER_VERTEX_SCRATCH: u32 = 1 << 0;
+const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1;
+const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2;
+const RENDER_RSRC_SPEC_HI: u32 = 1 << 4;
+const RENDER_DBIAS_IS_INT: u32 = 1 << 18;
+const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH
+    | RENDER_PROCESS_EMPTY_TILES
+    | RENDER_NO_VERTEX_CLUSTERING
+    | RENDER_RSRC_SPEC_HI
+    | RENDER_DBIAS_IS_INT;
+
+const SAMPLER_SIZE: u64 = 8;
+const MAX_SAMPLERS: u32 = 1024;
+const PAGE_SIZE: u64 = 0x4000;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) enum UapiTranslateError {
+    HeaderTruncated,
+    PayloadTruncated,
+    PayloadTail,
+    CommandType,
+    AttachmentPayloadSize,
+    AttachmentCount,
+    AttachmentFields,
+    AttachmentBarrier,
+    HardwareCommandCount,
+    BarrierFuture,
+    QueueUscAlignment,
+    QueueUscRange,
+    AddressRange,
+    AddressAlignment,
+    FlagsUnknownBits,
+    FlagsUnsupported,
+    Dimensions,
+    Layers,
+    UtileDimensions,
+    Samples,
+    TilebufferSize,
+    SamplerCount,
+    SamplerPair,
+    HelperProgram,
+    Add3ProofAttachments,
+    ProgramOffset,
+    ZlsFields,
+    StencilClearBits,
+}
+
+impl UapiTranslateError {
+    /// Stable short name for one rejection, so a failing submit reports which
+    /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no
+    /// `Debug` formatting) to stay cheap on the ioctl path.
+    pub(crate) fn name(&self) -> &'static str {
+        match self {
+            Self::HeaderTruncated => "header-truncated",
+            Self::PayloadTruncated => "payload-truncated",
+            Self::PayloadTail => "payload-tail",
+            Self::CommandType => "command-type",
+            Self::AttachmentPayloadSize => "attachment-payload-size",
+            Self::AttachmentCount => "attachment-count",
+            Self::AttachmentFields => "attachment-fields",
+            Self::AttachmentBarrier => "attachment-barrier",
+            Self::HardwareCommandCount => "hardware-command-count",
+            Self::BarrierFuture => "barrier-future",
+            Self::QueueUscAlignment => "queue-usc-alignment",
+            Self::QueueUscRange => "queue-usc-range",
+            Self::AddressRange => "address-range",
+            Self::AddressAlignment => "address-alignment",
+            Self::FlagsUnknownBits => "flags-unknown-bits",
+            Self::FlagsUnsupported => "flags-unsupported",
+            Self::Dimensions => "dimensions",
+            Self::Layers => "layers",
+            Self::UtileDimensions => "utile-dimensions",
+            Self::Samples => "samples",
+            Self::TilebufferSize => "tilebuffer-size",
+            Self::SamplerCount => "sampler-count",
+            Self::SamplerPair => "sampler-pair",
+            Self::HelperProgram => "helper-program",
+            Self::Add3ProofAttachments => "add3-proof-attachments",
+            Self::ProgramOffset => "program-offset",
+            Self::ZlsFields => "zls-fields",
+            Self::StencilClearBits => "stencil-clear-bits",
+
+
+        }
+    }
+}
+
+fn get_u16(raw: &[u8], offset: usize) -> u16 {
+    u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap())
+}
+
+fn get_u32(raw: &[u8], offset: usize) -> u32 {
+    u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap())
+}
+
+fn get_u64(raw: &[u8], offset: usize) -> u64 {
+    u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap())
+}
+
+fn padded_payload<const N: usize>(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> {
+    if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) {
+        return Err(UapiTranslateError::PayloadTail);
+    }
+    let mut out = [0u8; N];
+    let count = core::cmp::min(raw.len(), N);
+    out[..count].copy_from_slice(&raw[..count]);
+    Ok(out)
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiCommandHeader {
+    pub(crate) command_type: u16,
+    pub(crate) size: u16,
+    pub(crate) render_barrier: u16,
+    pub(crate) compute_barrier: u16,
+}
+
+impl UapiCommandHeader {
+    fn parse(raw: &[u8]) -> Result<Self, UapiTranslateError> {
+        if raw.len() < UAPI_COMMAND_HEADER_SIZE {
+            return Err(UapiTranslateError::HeaderTruncated);
+        }
+        Ok(Self {
+            command_type: get_u16(raw, 0),
+            size: get_u16(raw, 2),
+            render_barrier: get_u16(raw, 4),
+            compute_barrier: get_u16(raw, 6),
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
+pub(crate) struct UapiAttachment {
+    pub(crate) address: u64,
+    pub(crate) size: u64,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiAttachmentList {
+    pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS],
+    pub(crate) count: u8,
+}
+
+impl UapiAttachmentList {
+    pub(crate) const EMPTY: Self = Self {
+        entries: [UapiAttachment {
+            address: 0,
+            size: 0,
+        }; UAPI_MAX_ATTACHMENTS],
+        count: 0,
+    };
+
+    pub(crate) fn as_slice(&self) -> &[UapiAttachment] {
+        &self.entries[..self.count as usize]
+    }
+
+    fn parse(raw: &[u8]) -> Result<Self, UapiTranslateError> {
+        if raw.len() % UAPI_ATTACHMENT_SIZE != 0 {
+            return Err(UapiTranslateError::AttachmentPayloadSize);
+        }
+        let count = raw.len() / UAPI_ATTACHMENT_SIZE;
+        if count > UAPI_MAX_ATTACHMENTS {
+            return Err(UapiTranslateError::AttachmentCount);
+        }
+        let mut out = Self::EMPTY;
+        for index in 0..count {
+            let offset = index * UAPI_ATTACHMENT_SIZE;
+            if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 {
+                return Err(UapiTranslateError::AttachmentFields);
+            }
+            out.entries[index] = UapiAttachment {
+                address: get_u64(raw, offset),
+                size: get_u64(raw, offset + 8),
+            };
+        }
+        out.count = count as u8;
+        Ok(out)
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiHelperProgram {
+    pub(crate) binary: u32,
+    pub(crate) config: u32,
+    pub(crate) data: u64,
+}
+
+fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram {
+    UapiHelperProgram {
+        binary: get_u32(raw, offset),
+        config: get_u32(raw, offset + 4),
+        data: get_u64(raw, offset + 8),
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiProgram {
+    pub(crate) usc: u32,
+    pub(crate) resource_spec: u64,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiTimestamp {
+    pub(crate) handle: u32,
+    pub(crate) offset: u32,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiTimestamps {
+    pub(crate) start: UapiTimestamp,
+    pub(crate) end: UapiTimestamp,
+}
+
+fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps {
+    UapiTimestamps {
+        start: UapiTimestamp {
+            handle: get_u32(raw, offset),
+            offset: get_u32(raw, offset + 4),
+        },
+        end: UapiTimestamp {
+            handle: get_u32(raw, offset + 8),
+            offset: get_u32(raw, offset + 12),
+        },
+    }
+}
+
+fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram {
+    UapiProgram {
+        usc: get_u32(raw, offset),
+        resource_spec: get_u32(raw, offset + 4) as u64
+            | (get_u32(raw, high_offset) as u64) << 32,
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiZlsBuffer {
+    pub(crate) base: u64,
+    pub(crate) compression_base: u64,
+    pub(crate) stride: u32,
+    pub(crate) compression_stride: u32,
+}
+
+fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer {
+    UapiZlsBuffer {
+        base: get_u64(raw, offset),
+        compression_base: get_u64(raw, offset + 8),
+        stride: get_u32(raw, offset + 0x10),
+        compression_stride: get_u32(raw, offset + 0x14),
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiRenderCommand {
+    pub(crate) flags: u32,
+    pub(crate) depth_dimensions: u32,
+    pub(crate) vdm_base: u64,
+    pub(crate) vertex_helper: UapiHelperProgram,
+    pub(crate) fragment_helper: UapiHelperProgram,
+    pub(crate) scissor_base: u64,
+    pub(crate) depth_bias_base: u64,
+    pub(crate) occlusion_query_base: u64,
+    pub(crate) depth: UapiZlsBuffer,
+    pub(crate) stencil: UapiZlsBuffer,
+    pub(crate) zls_control: u64,
+    pub(crate) multisample_control: u64,
+    pub(crate) sampler_heap: u64,
+    pub(crate) ppp_control: u32,
+    pub(crate) width: u16,
+    pub(crate) height: u16,
+    pub(crate) layers: u16,
+    pub(crate) sampler_count: u16,
+    pub(crate) utile_width: u8,
+    pub(crate) utile_height: u8,
+    pub(crate) samples: u8,
+    pub(crate) sample_size: u8,
+    pub(crate) merge_upper_x: u32,
+    pub(crate) merge_upper_y: u32,
+    pub(crate) background: UapiProgram,
+    pub(crate) end_of_tile: UapiProgram,
+    pub(crate) partial_background: UapiProgram,
+    pub(crate) partial_end_of_tile: UapiProgram,
+    pub(crate) depth_clear: u32,
+    pub(crate) stencil_clear: u32,
+    pub(crate) vertex_timestamps: UapiTimestamps,
+    pub(crate) fragment_timestamps: UapiTimestamps,
+}
+
+impl UapiRenderCommand {
+    fn parse(raw: &[u8]) -> Result<Self, UapiTranslateError> {
+        let raw = padded_payload::<UAPI_RENDER_SIZE>(raw)?;
+        Ok(Self {
+            flags: get_u32(&raw, 0x00),
+            depth_dimensions: get_u32(&raw, 0x04),
+            vdm_base: get_u64(&raw, 0x08),
+            vertex_helper: helper(&raw, 0x10),
+            fragment_helper: helper(&raw, 0x20),
+            scissor_base: get_u64(&raw, 0x30),
+            depth_bias_base: get_u64(&raw, 0x38),
+            occlusion_query_base: get_u64(&raw, 0x40),
+            depth: zls(&raw, 0x48),
+            stencil: zls(&raw, 0x60),
+            zls_control: get_u64(&raw, 0x78),
+            multisample_control: get_u64(&raw, 0x80),
+            sampler_heap: get_u64(&raw, 0x88),
+            ppp_control: get_u32(&raw, 0x90),
+            width: get_u16(&raw, 0x94),
+            height: get_u16(&raw, 0x96),
+            layers: get_u16(&raw, 0x98),
+            sampler_count: get_u16(&raw, 0x9a),
+            utile_width: raw[0x9c],
+            utile_height: raw[0x9d],
+            samples: raw[0x9e],
+            sample_size: raw[0x9f],
+            merge_upper_x: get_u32(&raw, 0xa0),
+            merge_upper_y: get_u32(&raw, 0xa4),
+            background: program(&raw, 0xa8, 0xf8),
+            end_of_tile: program(&raw, 0xb0, 0xf0),
+            partial_background: program(&raw, 0xb8, 0xfc),
+            partial_end_of_tile: program(&raw, 0xc0, 0xf4),
+            depth_clear: get_u32(&raw, 0xc8),
+            stencil_clear: get_u32(&raw, 0xcc),
+            vertex_timestamps: timestamps(&raw, 0xd0),
+            fragment_timestamps: timestamps(&raw, 0xe0),
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct UapiComputeCommand {
+    pub(crate) flags: u32,
+    pub(crate) sampler_count: u32,
+    pub(crate) control_stream_base: u64,
+    pub(crate) control_stream_end: u64,
+    pub(crate) sampler_heap: u64,
+    pub(crate) helper: UapiHelperProgram,
+    pub(crate) timestamps: UapiTimestamps,
+}
+
+impl UapiComputeCommand {
+    fn parse(raw: &[u8]) -> Result<Self, UapiTranslateError> {
+        let raw = padded_payload::<UAPI_COMPUTE_SIZE>(raw)?;
+        Ok(Self {
+            flags: get_u32(&raw, 0),
+            sampler_count: get_u32(&raw, 4),
+            control_stream_base: get_u64(&raw, 8),
+            control_stream_end: get_u64(&raw, 0x10),
+            sampler_heap: get_u64(&raw, 0x18),
+            helper: helper(&raw, 0x20),
+            timestamps: timestamps(&raw, 0x30),
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) enum ParsedHardwareCommand {
+    Render {
+        index: u32,
+        header: UapiCommandHeader,
+        payload: UapiRenderCommand,
+        vertex_attachments: UapiAttachmentList,
+        fragment_attachments: UapiAttachmentList,
+    },
+    Compute {
+        index: u32,
+        header: UapiCommandHeader,
+        payload: UapiComputeCommand,
+        attachments: UapiAttachmentList,
+    },
+}
+
+pub(crate) struct UapiCommandParser<'a> {
+    raw: &'a [u8],
+    offset: usize,
+    command_index: u32,
+    hardware_count: u32,
+    render_count: u16,
+    compute_count: u16,
+    vertex_attachments: UapiAttachmentList,
+    fragment_attachments: UapiAttachmentList,
+    compute_attachments: UapiAttachmentList,
+}
+
+impl<'a> UapiCommandParser<'a> {
+    pub(crate) const fn new(raw: &'a [u8]) -> Self {
+        Self {
+            raw,
+            offset: 0,
+            command_index: 0,
+            hardware_count: 0,
+            render_count: 0,
+            compute_count: 0,
+            vertex_attachments: UapiAttachmentList::EMPTY,
+            fragment_attachments: UapiAttachmentList::EMPTY,
+            compute_attachments: UapiAttachmentList::EMPTY,
+        }
+    }
+
+    pub(crate) fn finish(&self) -> Result<u32, UapiTranslateError> {
+        if self.offset != self.raw.len() {
+            return Err(UapiTranslateError::PayloadTruncated);
+        }
+        if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS {
+            return Err(UapiTranslateError::HardwareCommandCount);
+        }
+        Ok(self.hardware_count)
+    }
+
+    pub(crate) fn next_hardware(
+        &mut self,
+    ) -> Result<Option<ParsedHardwareCommand>, UapiTranslateError> {
+        loop {
+            if self.offset == self.raw.len() {
+                return Ok(None);
+            }
+            if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE {
+                return Err(UapiTranslateError::HeaderTruncated);
+            }
+            let header = UapiCommandHeader::parse(&self.raw[self.offset..])?;
+            self.offset += UAPI_COMMAND_HEADER_SIZE;
+            let end = self
+                .offset
+                .checked_add(header.size as usize)
+                .ok_or(UapiTranslateError::PayloadTruncated)?;
+            if end > self.raw.len() {
+                return Err(UapiTranslateError::PayloadTruncated);
+            }
+            let payload = &self.raw[self.offset..end];
+            self.offset = end;
+            self.command_index += 1;
+
+            match header.command_type {
+                COMMAND_RENDER | COMMAND_COMPUTE => {
+                    if header.render_barrier != UAPI_BARRIER_NONE
+                        && header.render_barrier > self.render_count
+                    {
+                        return Err(UapiTranslateError::BarrierFuture);
+                    }
+                    if header.compute_barrier != UAPI_BARRIER_NONE
+                        && header.compute_barrier > self.compute_count
+                    {
+                        return Err(UapiTranslateError::BarrierFuture);
+                    }
+                    self.hardware_count += 1;
+                    if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS {
+                        return Err(UapiTranslateError::HardwareCommandCount);
+                    }
+                    if header.command_type == COMMAND_RENDER {
+                        self.render_count += 1;
+                        return Ok(Some(ParsedHardwareCommand::Render {
+                            index: self.command_index,
+                            header,
+                            payload: UapiRenderCommand::parse(payload)?,
+                            vertex_attachments: self.vertex_attachments,
+                            fragment_attachments: self.fragment_attachments,
+                        }));
+                    }
+                    self.compute_count += 1;
+                    return Ok(Some(ParsedHardwareCommand::Compute {
+                        index: self.command_index,
+                        header,
+                        payload: UapiComputeCommand::parse(payload)?,
+                        attachments: self.compute_attachments,
+                    }));
+                }
+                SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => {
+                    if header.render_barrier != UAPI_BARRIER_NONE
+                        || header.compute_barrier != UAPI_BARRIER_NONE
+                    {
+                        return Err(UapiTranslateError::AttachmentBarrier);
+                    }
+                    let attachments = UapiAttachmentList::parse(payload)?;
+                    match header.command_type {
+                        SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments,
+                        SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments,
+                        SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments,
+                        _ => unreachable!(),
+                    }
+                }
+                _ => return Err(UapiTranslateError::CommandType),
+            }
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) enum GpuAccess {
+    Read,
+    Write,
+    ReadWrite,
+}
+
+pub(crate) trait GpuAddressSpace {
+    fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool;
+}
+
+fn require_range<A: GpuAddressSpace>(
+    address_space: &A,
+    address: u64,
+    size: u64,
+    access: GpuAccess,
+) -> Result<(), UapiTranslateError> {
+    let end = address
+        .checked_add(size)
+        .ok_or(UapiTranslateError::AddressRange)?;
+    if size == 0 || end <= address || !address_space.covers(address, size, access) {
+        return Err(UapiTranslateError::AddressRange);
+    }
+    Ok(())
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct QueueUscWindow {
+    pub(crate) base: u64,
+    pub(crate) user_start: u64,
+    pub(crate) user_end: u64,
+}
+
+impl QueueUscWindow {
+    pub(crate) fn validate(self) -> Result<(), UapiTranslateError> {
+        if self.base & (USC_WINDOW_SIZE - 1) != 0 {
+            return Err(UapiTranslateError::QueueUscAlignment);
+        }
+        let end = self
+            .base
+            .checked_add(USC_WINDOW_SIZE)
+            .ok_or(UapiTranslateError::QueueUscRange)?;
+        if self.base < self.user_start || end > self.user_end || end <= self.base {
+            return Err(UapiTranslateError::QueueUscRange);
+        }
+        Ok(())
+    }
+
+    pub(crate) fn full_program<A: GpuAddressSpace>(
+        self,
+        address_space: &A,
+        tagged_offset: u32,
+    ) -> Result<u64, UapiTranslateError> {
+        self.validate()?;
+        if tagged_offset == 0 {
+            return Err(UapiTranslateError::ProgramOffset);
+        }
+        let allocation = self
+            .base
+            .checked_add((tagged_offset & !7) as u64)
+            .ok_or(UapiTranslateError::ProgramOffset)?;
+        require_range(address_space, allocation, 4, GpuAccess::Read)?;
+        Ok(allocation)
+    }
+}
+
+fn validate_sampler<A: GpuAddressSpace>(
+    address_space: &A,
+    heap: u64,
+    count: u32,
+) -> Result<(), UapiTranslateError> {
+    if count > MAX_SAMPLERS {
+        return Err(UapiTranslateError::SamplerCount);
+    }
+    if (heap == 0) != (count == 0) {
+        return Err(UapiTranslateError::SamplerPair);
+    }
+    if count != 0 {
+        if heap & (SAMPLER_SIZE - 1) != 0 {
+            return Err(UapiTranslateError::AddressAlignment);
+        }
+        require_range(
+            address_space,
+            heap,
+            count as u64 * SAMPLER_SIZE,
+            GpuAccess::Read,
+        )?;
+    }
+    Ok(())
+}
+
+pub(crate) fn validate_attachments<A: GpuAddressSpace>(
+    address_space: &A,
+    attachments: &UapiAttachmentList,
+) -> Result<(), UapiTranslateError> {
+    for attachment in attachments.as_slice() {
+        require_range(
+            address_space,
+            attachment.address,
+            attachment.size,
+            GpuAccess::Write,
+        )?;
+    }
+    Ok(())
+}
+
+pub(crate) fn validate_zls<A: GpuAddressSpace>(
+    address_space: &A,
+    value: UapiZlsBuffer,
+    layers: u16,
+) -> Result<(), UapiTranslateError> {
+    if value.base == 0
+        && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0)
+    {
+        return Err(UapiTranslateError::ZlsFields);
+    }
+    if value.compression_base == 0 && value.compression_stride != 0 {
+        return Err(UapiTranslateError::ZlsFields);
+    }
+    if layers > 1 && value.base != 0 && value.stride == 0 {
+        return Err(UapiTranslateError::ZlsFields);
+    }
+    if value.stride != 0 && value.stride & 0x3fff != 1 {
+        return Err(UapiTranslateError::ZlsFields);
+    }
+    if value.compression_stride & 0x3fff != 0 {
+        return Err(UapiTranslateError::ZlsFields);
+    }
+    if value.base != 0 {
+        let layer_stride = if value.stride == 0 {
+            0
+        } else {
+            ((value.stride as u64 >> 14) + 1) * PAGE_SIZE
+        };
+        let size = layer_stride
+            .checked_mul(layers.saturating_sub(1) as u64)
+            .and_then(|value| value.checked_add(1))
+            .ok_or(UapiTranslateError::AddressRange)?;
+        require_range(address_space, value.base, size, GpuAccess::ReadWrite)?;
+    }
+    if value.compression_base != 0 {
+        let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80;
+        let size = layer_stride
+            .checked_mul(layers.saturating_sub(1) as u64)
+            .and_then(|value| value.checked_add(1))
+            .ok_or(UapiTranslateError::AddressRange)?;
+        require_range(
+            address_space,
+            value.compression_base,
+            size,
+            GpuAccess::ReadWrite,
+        )?;
+    }
+    Ok(())
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) struct TranslatedComputeCommand {
+    pub(crate) usc_exec_base: u64,
+    pub(crate) control_stream_base: u64,
+    pub(crate) control_stream_end: u64,
+    pub(crate) control_stream_terminator: u64,
+    pub(crate) sampler_heap: u64,
+    pub(crate) sampler_count: u32,
+    pub(crate) attachments: UapiAttachmentList,
+    pub(crate) timestamps: UapiTimestamps,
+}
+
+pub(crate) fn translate_compute_command<A: GpuAddressSpace>(
+    command: UapiComputeCommand,
+    attachments: UapiAttachmentList,
+    queue: QueueUscWindow,
+    address_space: &A,
+) -> Result<TranslatedComputeCommand, UapiTranslateError> {
+    queue.validate()?;
+    if command.flags != 0 {
+        return Err(UapiTranslateError::FlagsUnknownBits);
+    }
+    if command.control_stream_base & 3 != 0
+        || command.control_stream_end & 3 != 0
+        || command.control_stream_end <= command.control_stream_base
+    {
+        return Err(UapiTranslateError::AddressAlignment);
+    }
+    require_range(
+        address_space,
+        command.control_stream_base,
+        command.control_stream_end - command.control_stream_base,
+        GpuAccess::Read,
+    )?;
+    validate_sampler(address_space, command.sampler_heap, command.sampler_count)?;
+    if command.helper
+        != (UapiHelperProgram {
+            binary: 0,
+            config: 0,
+            data: 0,
+        })
+    {
+        return Err(UapiTranslateError::HelperProgram);
+    }
+    Ok(TranslatedComputeCommand {
+        usc_exec_base: queue.base,
+        control_stream_base: command.control_stream_base,
+        control_stream_end: command.control_stream_end,
+        control_stream_terminator: command.control_stream_end - 4,
+        sampler_heap: command.sampler_heap,
+        sampler_count: command.sampler_count,
+        attachments,
+        timestamps: command.timestamps,
+    })
+}
diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs
new file mode 100644
index 0000000000000..785b7f2314c8f
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs
@@ -0,0 +1,1784 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! GPU initialization / global structures
+
+use crate::fw::channels;
+use crate::fw::types::*;
+use crate::{
+    default_zeroed,
+    gem,
+    mmu,
+    no_debug,
+    trivial_gpustruct, //
+};
+
+pub(crate) mod raw {
+    use super::*;
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct ChannelRing<T: GpuStruct + Debug + Default, U: Copy> {
+        pub(crate) state: Option<GpuWeakPointer<T>>,
+        pub(crate) ring: Option<GpuWeakPointer<[U]>>,
+    }
+
+    /// Host->FW ring descriptor of the 14.x firmware ABI.
+    ///
+    /// Instead of one `ChannelState` pointer, each host->FW ring (pipes, DeviceControl) is
+    /// described by the FW VAs of its three index words and of the ring itself. On G15 the
+    /// indices (and the pipe rings) live in Fender scratch SRAM.
+    #[derive(Debug, Default, Copy, Clone)]
+    #[repr(C)]
+    pub(crate) struct ScratchRingDesc {
+        /// FW VA of the u32 read index (FW writes; scratch part B). +0x00
+        pub(crate) read_index: U64,
+        /// FW VA of the u32 "CFI" index (FW writes; scratch part B). +0x08
+        pub(crate) cfi_index: U64,
+        /// FW VA of the u32 write index (host writes; scratch part A). +0x10
+        pub(crate) write_index: U64,
+        /// FW VA of the ring entries. +0x18
+        pub(crate) ring: U64,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct PipeChannelsG15V14_8_3 {
+
+        // 14.x ABI: {TA, 3D, CL} descriptors of 0x20 bytes, 0x60 per priority
+        pub(crate) vtx: ScratchRingDesc,
+        pub(crate) frag: ScratchRingDesc,
+        pub(crate) comp: ScratchRingDesc,
+    }
+    default_zeroed!(PipeChannelsG15V14_8_3);
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct FwStatusFlags {
+        pub(crate) halt_count: AtomicU64,
+        __pad0: Pad<0x8>,
+        pub(crate) halted: AtomicU32,
+        __pad1: Pad<0xc>,
+        pub(crate) resume: AtomicU32,
+        __pad2: Pad<0xc>,
+        pub(crate) unk_40: u32,
+        __pad3: Pad<0xc>,
+        pub(crate) unk_ctr: u32,
+        /// P0+0x45c4. Non-zero makes the firmware install its work-scheduler callbacks
+        /// Left 0, the first compute kick branches to address 0 (observed on J516S).
+        pub(crate) sched_callbacks: u32,
+        __pad4: Pad<0x8>,
+        pub(crate) unk_60: u32,
+        __pad5: Pad<0xc>,
+        pub(crate) unk_70: u32,
+        __pad6: Pad<0xc>,
+    }
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct FwStatus {
+        pub(crate) fwctl_channel: ChannelRing<channels::FwCtlChannelState, channels::FwCtlMsg>,
+        pub(crate) flags: FwStatusFlags,
+    }
+
+    /// G15 "P0" globals-2/status block, 0xc3d0 bytes, pointed to by InitData+0xb0. It embeds the
+    /// G13 `FwStatus` at +0x4568 (FWCtl ring pointers, 8 bytes
+    /// of padding, then the halt/resume flags at +0x4580), which is why the
+    /// FWCtl ring and flags keep their `FwStatus` field names here.
+    #[repr(C)]
+    pub(crate) struct StatusBlock {
+        /// 0 at init; also written by the firmware register dump. +0x0
+        pub(crate) unk_0: u32,
+        /// Noise-suppression idle power-off state. +0x4
+        pub(crate) noise_suppression_idle_pwroff: u32,
+        /// 0 at init. +0x8
+        pub(crate) unk_8: u32,
+        /// Not touched by the host (0x0008..0x4017).
+        /// TODO: firmware-private, or tables (pending stamps?) the host must fill?
+        pub(crate) unk_c: Array<0x400c, u8>,
+        /// Perf-counter sampler control, 0x4018..0x40a0. Unused by drm/asahi.
+        pub(crate) perfctr: Array<0x88, u8>,
+        /// FW-initiated recovery info (FW-written). +0x40a0/+0x40a4
+        pub(crate) fw_recovery_info_a: u32,
+        pub(crate) fw_recovery_info_b: u32,
+        pub(crate) unk_40a8: Array<0x3f8, u8>,
+        /// Recovery-packet details read on event type 4. +0x44a0
+        pub(crate) recovery_packet_info: Array<0x14, u8>,
+        pub(crate) unk_44b4: Array<0xc, u8>,
+        /// Read while draining events. +0x44c0
+        pub(crate) unk_44c0: U64,
+        pub(crate) unk_44c8: Array<0x90, u8>,
+        /// Read while draining events. +0x4558/+0x4560
+        pub(crate) unk_4558: U64,
+        pub(crate) unk_4560: U64,
+        /// FWCtl `{state, ring}` FW VAs. +0x4568/+0x4570
+        pub(crate) fwctl_channel: ChannelRing<channels::FwCtlChannelState, channels::FwCtlMsg>,
+        /// 8 bytes of padding put the flags on a 16-byte boundary.
+        pub(crate) __pad_4578: Pad<0x8>,
+        /// halt_count/halted/resume/... at +0x4580/+0x4590/+0x45a0.
+        /// +0x45c4 (`flags.sched_callbacks`) must be non-zero or the work scheduler
+        /// is left without its callbacks.
+        pub(crate) flags: FwStatusFlags,
+        /// Not touched by the host (0x45c8..0xc3c7).
+        pub(crate) unk_45f0: Array<0x7dd8, u8>,
+        /// QoS mode, 0 at init. +0xc3c8
+        pub(crate) qos_mode: u32,
+        /// TODO: read-modify-written at shared-data init; left 0. +0xc3cc
+        pub(crate) unk_c3cc: u32,
+    }
+    default_zeroed!(StatusBlock);
+    no_debug!(StatusBlock);
+
+    /// G15 "P1" power-controller shared block, 0x238 bytes, InitData+0xb8. Its 0x0c..0xd4 part
+    /// keeps the order of the G13 13.5 `Globals` power
+    /// fields (power zones, fast-die and PPM controllers), so the same field names are used.
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct PowerCtlBlock {
+        /// FW-published power values. +0x0/+0x4
+        pub(crate) fw_power_a: u32,
+        pub(crate) fw_power_b: u32,
+        /// Power-zone reset request. +0x8
+        pub(crate) power_zone_reset: u32,
+        /// +0xc..+0x5f: G13 Globals `power_zone_count`..`power_zones`, same order.
+        pub(crate) power_zone_count: u32,
+        pub(crate) avg_power_filter_tc_periods: u32,
+        pub(crate) avg_power_ki_dt: F32,
+        pub(crate) avg_power_kp: F32,
+        pub(crate) avg_power_min_duty_cycle: u32,
+        pub(crate) avg_power_target_filter_tc: u32,
+        pub(crate) power_zones: Array<5, PowerZoneGlobal>,
+        /// CLVR loop override / control effort. +0x60..+0x68
+        pub(crate) clvr_loop_override_a: u32,
+        pub(crate) clvr_ctrl_effort: u32,
+        pub(crate) clvr_loop_override_b: u32,
+        pub(crate) unk_6c: Array<0x38, u8>,
+        /// Lifetime-servo temperature target. +0xa4
+        pub(crate) lifetime_servo_temp_a: u32,
+        /// +0xa8..+0xc8: G13 Globals `unk_89bc`..`unk_89e0`, same order.
+        pub(crate) fast_die_temp_target: u32,
+        pub(crate) fast_die0_release_temp: u32,
+        pub(crate) unk_b0: i32,
+        pub(crate) fast_die0_prop_tgt_delta: u32,
+        pub(crate) fast_die0_kp: F32,
+        pub(crate) fast_die0_ki_dt: F32,
+        pub(crate) unk_c0: u32,
+        pub(crate) unk_c4: u32,
+        /// +0xc8..+0xdb: the fast-die/PPM enable flags, the PPM target (maximum power) and the
+        /// PPM controller gains, in G13 Globals order (`unk_89e0`, `max_power_2`, `ppm_kp`,
+        /// `ppm_ki_dt`). The two T6030 backends place them differently: the manager starts
+        /// with the enable flag at +0xc8 (the layout it was validated with), the runtime backend
+        /// with 0 at +0xc8 and the flag at +0xcc, as its InitData has them. See the builder.
+        pub(crate) ppm_words: Array<5, u32>,
+        pub(crate) unk_dc: Array<0xc, u8>,
+        /// "SE" controller override / engagement enable. +0xe8/+0xec
+        pub(crate) se_controller_override: u32,
+        pub(crate) se_engagement: u32,
+        pub(crate) unk_f0: Array<0x68, u8>,
+        /// DPE leakage update config. +0x158..+0x1ab
+        pub(crate) dpe_leak_cfg_a: u32,
+        pub(crate) unk_15c: u32,
+        pub(crate) unk_160: u32,
+        pub(crate) unk_164: u32,
+        pub(crate) dpe_leak_coefs: Array<0x40, u8>,
+        pub(crate) dpe_leak_cfg_b: u32,
+        pub(crate) unk_1ac: Array<0x8, u8>,
+        /// CLPC shared data (FW-written). +0x1b4..+0x1cc
+        pub(crate) clpc_shared_a: U64,
+        pub(crate) clpc_shared_b: U64,
+        pub(crate) clpc_split_ratio_a: u32,
+        pub(crate) clpc_split_ratio_b: u32,
+        pub(crate) clpc_shared_c: u32,
+        /// +0x1d0 (u8) / +0x1d1 (unaligned u32)
+        pub(crate) perf_ctrl_override_active: u8,
+        pub(crate) consistent_perf_state: U32,
+        pub(crate) unk_1d5: Array<0x3, u8>,
+        /// FW-written energy / filtered power. +0x1d8/+0x1e0/+0x1e4
+        pub(crate) accumulated_energy: U64,
+        pub(crate) filtered_power: u32,
+        pub(crate) unk_1e4: u32,
+        /// SLC DSID configuration. +0x1e8..+0x22b
+        pub(crate) dsid_cfg_a: U64,
+        pub(crate) unk_1f0: Array<0x28, u8>,
+        pub(crate) dsid_cfg_b: U64,
+        pub(crate) dsid_cfg_c: U64,
+        pub(crate) dsid_cfg_d: u32,
+        pub(crate) unk_22c: u32,
+        /// +0x230
+        pub(crate) submit_in_progress_gate: u32,
+        pub(crate) unk_234: u32,
+    }
+    default_zeroed!(PowerCtlBlock);
+
+    /// G15 "P2" debug/trace block, 0x20 bytes, InitData+0xa8.
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct DebugBlock {
+        /// Moved here from G13 `Globals.ktrace_enable`. +0x0
+        pub(crate) ktrace_enable: u32,
+        /// TODO: written at init with an unknown value; left 0. +0x4
+        pub(crate) unk_4: u32,
+        /// System-sleep notification in progress. +0x8
+        pub(crate) system_sleep_in_progress: u32,
+        pub(crate) unk_c: u32,
+        /// Firmware-written. Read-only for the host.
+        pub(crate) fw_word_10: AtomicU32,
+        /// Firmware init progress: the firmware writes 0 once it consumes the INIT handshake and 1
+        /// once its post-INIT initialisation is done, each followed by a barrier. The G15 driver
+        /// writes a sentinel here before MSG_INIT so the three states (not consumed / stuck / done)
+        /// are distinguishable.
+        pub(crate) init_state: AtomicU32,
+        pub(crate) unk_18: u32,
+        pub(crate) unk_1c: u32,
+    }
+    default_zeroed!(DebugBlock);
+
+    /// One Globals register-override record, 0x18 bytes.
+    #[derive(Debug, Default, Copy, Clone)]
+    #[repr(C)]
+    pub(crate) struct RegisterOverride {
+        pub(crate) addr: U64,
+        pub(crate) value: U64,
+        pub(crate) id: u32,
+        pub(crate) __pad: u32,
+    }
+
+    /// One InitSeq register-initialisation record, 0x18 bytes.
+    ///
+    /// The InitSeq buffer (InitData+0x08) is a list of these. A zeroed buffer is a valid empty
+    /// list, which is what drm/asahi passes; this type only documents the format.
+    #[derive(Debug, Default, Copy, Clone)]
+    #[repr(C)]
+    #[allow(dead_code)]
+    pub(crate) struct InitSeqRecord {
+        pub(crate) value: U64,
+        pub(crate) reg_id: u32,
+        pub(crate) extra: u32,
+        /// 1 = 32-bit, 2 = 64-bit, 3 = 64-bit physical address.
+        pub(crate) kind: u32,
+        pub(crate) __pad: u32,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataShared1 {
+        pub(crate) table: Array<16, i32>,
+        pub(crate) unk_44: Array<0x60, u8>,
+        pub(crate) unk_a4: u32,
+        pub(crate) unk_a8: u32,
+    }
+    default_zeroed!(HwDataShared1);
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct HwDataShared2Curve {
+        pub(crate) unk_0: u32,
+        pub(crate) unk_4: u32,
+        pub(crate) t1: Array<16, u16>,
+        pub(crate) t2: Array<16, i16>,
+        pub(crate) t3: Array<8, Array<16, i32>>,
+    }
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct HwDataShared2G14 {
+        pub(crate) unk_0: Array<5, u32>,
+        pub(crate) unk_14: u32,
+        pub(crate) unk_18: Array<8, u32>,
+        pub(crate) curve1: HwDataShared2Curve,
+        pub(crate) curve2: HwDataShared2Curve,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataShared2 {
+        pub(crate) table: Array<10, i32>,
+        pub(crate) unk_28: Array<0x10, u8>,
+        pub(crate) g14: HwDataShared2G14,
+        pub(crate) unk_500: u32,
+        pub(crate) unk_504: u32,
+        pub(crate) unk_508: u32,
+        pub(crate) unk_50c: u32,
+    }
+    default_zeroed!(HwDataShared2);
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataShared3 {
+        pub(crate) unk_0: u32,
+        pub(crate) unk_4: u32,
+        pub(crate) unk_8: u32,
+        pub(crate) table: Array<16, u32>,
+        pub(crate) unk_4c: u32,
+    }
+    default_zeroed!(HwDataShared3);
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataA130Extra {
+        pub(crate) unk_0: Array<0x38, u8>,
+        pub(crate) unk_38: u32,
+        pub(crate) unk_3c: u32,
+        pub(crate) gpu_se_inactive_threshold: u32,
+        pub(crate) unk_44: u32,
+        pub(crate) gpu_se_engagement_criteria: i32,
+        pub(crate) gpu_se_reset_criteria: u32,
+        pub(crate) unk_50: u32,
+        pub(crate) unk_54: u32,
+        pub(crate) unk_58: u32,
+        pub(crate) unk_5c: u32,
+        pub(crate) gpu_se_filter_a_neg: F32,
+        pub(crate) gpu_se_filter_1_a_neg: F32,
+        pub(crate) gpu_se_filter_a: F32,
+        pub(crate) gpu_se_filter_1_a: F32,
+        pub(crate) gpu_se_ki_dt: F32,
+        pub(crate) gpu_se_ki_1_dt: F32,
+        pub(crate) unk_78: F32,
+        pub(crate) unk_7c: F32,
+        pub(crate) gpu_se_kp: F32,
+        pub(crate) gpu_se_kp_1: F32,
+        pub(crate) unk_88: u32,
+        pub(crate) unk_8c: u32,
+        pub(crate) max_pstate_scaled_1: u32,
+        pub(crate) unk_94: u32,
+        pub(crate) unk_98: u32,
+        pub(crate) unk_9c: F32,
+        pub(crate) unk_a0: u32,
+        pub(crate) unk_a4: u32,
+        pub(crate) gpu_se_filter_time_constant_ms: u32,
+        pub(crate) gpu_se_filter_time_constant_1_ms: u32,
+        pub(crate) gpu_se_filter_time_constant_clks: U64,
+        pub(crate) gpu_se_filter_time_constant_1_clks: U64,
+        pub(crate) unk_c0: u32,
+        pub(crate) unk_c4: F32,
+        pub(crate) unk_c8: Array<0x4c, u8>,
+        pub(crate) unk_114: F32,
+        pub(crate) unk_118: u32,
+        pub(crate) unk_11c: u32,
+        pub(crate) unk_120: u32,
+        pub(crate) unk_124: u32,
+        pub(crate) max_pstate_scaled_2: u32,
+        pub(crate) unk_12c: Array<0x8c, u8>,
+    }
+    default_zeroed!(HwDataA130Extra);
+
+    /// G15 (14.x) shader-engine controller block of HwDataA, at +0x10b4.
+    ///
+    /// It holds the fields of [`HwDataA130Extra`] in a different order: the filter, gain and
+    /// time-constant words move 0x14 bytes down, and the threshold/criteria header moves behind
+    /// them, to +0xd8. Offsets in the field names are G15 block offsets.
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataA140Extra {
+        pub(crate) unk_0: Array<0x40, u8>,
+        pub(crate) unk_40: u32,
+        pub(crate) unk_44: u32,
+        pub(crate) unk_48: u32,
+        pub(crate) gpu_se_filter_a_neg: F32,
+        pub(crate) gpu_se_filter_1_a_neg: F32,
+        pub(crate) gpu_se_filter_a: F32,
+        pub(crate) gpu_se_filter_1_a: F32,
+        pub(crate) gpu_se_ki_dt: F32,
+        pub(crate) gpu_se_ki_1_dt: F32,
+        pub(crate) unk_64: F32,
+        pub(crate) unk_68: F32,
+        pub(crate) gpu_se_kp: F32,
+        pub(crate) gpu_se_kp_1: F32,
+        pub(crate) unk_74: u32,
+        pub(crate) unk_78: u32,
+        pub(crate) max_pstate_scaled_1: u32,
+        pub(crate) min_pstate_scaled: u32,
+        pub(crate) unk_84: u32,
+        pub(crate) unk_88: F32,
+        /// Shader-engine target (ADT gpu-se-tgt).
+        pub(crate) se_target: u32,
+        pub(crate) unk_90: u32,
+        pub(crate) gpu_se_filter_time_constant_ms: u32,
+        pub(crate) gpu_se_filter_time_constant_1_ms: u32,
+        pub(crate) gpu_se_filter_time_constant_clks: U64,
+        pub(crate) gpu_se_filter_time_constant_1_clks: U64,
+        pub(crate) unk_ac: u32,
+        pub(crate) unk_b0: F32,
+        pub(crate) unk_b4: Array<0x24, u8>,
+        pub(crate) unk_d8: u32,
+        pub(crate) unk_dc: u32,
+        pub(crate) gpu_se_inactive_threshold: u32,
+        pub(crate) gpu_se_engagement_criteria: i32,
+        pub(crate) unk_e8: u32,
+        pub(crate) unk_ec: u32,
+        pub(crate) gpu_se_reset_criteria: u32,
+        pub(crate) unk_f4: Array<0x28, u8>,
+        pub(crate) unk_11c: F32,
+        pub(crate) unk_120: Array<0xc, u8>,
+        pub(crate) unk_12c: u32,
+        pub(crate) max_pstate_scaled_2: u32,
+        pub(crate) unk_134: Array<0x84, u8>,
+    }
+    default_zeroed!(HwDataA140Extra);
+
+    #[repr(C)]
+    pub(crate) struct T81xxData {
+        pub(crate) unk_d8c: u32,
+        pub(crate) unk_d90: u32,
+        pub(crate) unk_d94: u32,
+        pub(crate) unk_d98: u32,
+        pub(crate) unk_d9c: F32,
+        pub(crate) unk_da0: u32,
+        pub(crate) unk_da4: F32,
+        pub(crate) unk_da8: u32,
+        pub(crate) unk_dac: F32,
+        pub(crate) unk_db0: u32,
+        pub(crate) unk_db4: u32,
+        pub(crate) unk_db8: F32,
+        pub(crate) unk_dbc: F32,
+        pub(crate) unk_dc0: u32,
+        pub(crate) unk_dc4: u32,
+        pub(crate) unk_dc8: u32,
+        pub(crate) max_pstate_scaled: u32,
+    }
+    default_zeroed!(T81xxData);
+
+    #[derive(Default, Copy, Clone)]
+    #[repr(C)]
+    pub(crate) struct PowerZoneG15V14_8_3 {
+        pub(crate) val: F32,
+        pub(crate) target: u32,
+        pub(crate) target_off: u32,
+        pub(crate) filter_tc_x4: u32,
+        pub(crate) filter_tc_xperiod: u32,
+        pub(crate) unk_10: u32,
+        pub(crate) unk_14: u32,
+        pub(crate) filter_a_neg: F32,
+        pub(crate) filter_a: F32,
+        pub(crate) pad: u32,
+    }
+
+    // G15 HwDataA uses the G13 single-die layout with 8x8 coefficient tables at 0x970-0xb6f
+    //, so the G14X 16-core layout is G14X-only.
+    const MAX_CORES_PER_CLUSTERG15V14_8_3: usize = {
+        {
+            8
+        }
+    };
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct AuxLeakCoef {
+        pub(crate) afr_1: Array<2, F32>,
+        pub(crate) cs_1: Array<2, F32>,
+        pub(crate) afr_2: Array<2, F32>,
+        pub(crate) cs_2: Array<2, F32>,
+    }
+
+    #[repr(C)]
+    pub(crate) struct HwDataAG15V14_8_3 {
+        pub(crate) unk_0: u32,
+        pub(crate) clocks_per_period: u32,
+
+        pub(crate) clocks_per_period_2: u32,
+
+        pub(crate) unk_8: u32,
+        pub(crate) pwr_status: AtomicU32,
+        pub(crate) unk_10: F32,
+        pub(crate) unk_14: u32,
+        pub(crate) unk_18: u32,
+        pub(crate) unk_1c: u32,
+        pub(crate) unk_20: u32,
+        pub(crate) unk_24: u32,
+        pub(crate) actual_pstate: u32,
+        pub(crate) tgt_pstate: u32,
+        pub(crate) unk_30: u32,
+        pub(crate) cur_pstate: u32,
+        pub(crate) unk_38: u32,
+
+        pub(crate) unk_3c_0: u32,
+
+        pub(crate) base_pstate_scaled: u32,
+        pub(crate) unk_40: u32,
+        pub(crate) max_pstate_scaled: u32,
+        pub(crate) unk_48: u32,
+        pub(crate) min_pstate_scaled: u32,
+        pub(crate) freq_mhz: F32,
+        pub(crate) unk_54: Array<0x20, u8>,
+
+        pub(crate) unk_74_0: u32,
+
+        pub(crate) sram_k: Array<0x10, F32>,
+        pub(crate) unk_b4: Array<0x100, u8>,
+        pub(crate) unk_1b4: u32,
+        pub(crate) temp_c: u32,
+        pub(crate) avg_power_mw: u32,
+        pub(crate) update_ts: U64,
+        pub(crate) unk_1c8: u32,
+        pub(crate) unk_1cc: Array<0x478, u8>,
+        pub(crate) pad_644: Pad<0x8>,
+        pub(crate) unk_64c: u32,
+        pub(crate) unk_650: u32,
+        pub(crate) pad_654: u32,
+        pub(crate) pwr_filter_a_neg: F32,
+        pub(crate) pad_65c: u32,
+        pub(crate) pwr_filter_a: F32,
+        pub(crate) pad_664: u32,
+        pub(crate) pwr_integral_gain: F32,
+        pub(crate) pad_66c: u32,
+        pub(crate) pwr_integral_min_clamp: F32,
+        pub(crate) max_power_1: F32,
+        pub(crate) pwr_proportional_gain: F32,
+        pub(crate) pad_67c: u32,
+        pub(crate) pwr_pstate_related_k: F32,
+        pub(crate) pwr_pstate_max_dc_offset: i32,
+        pub(crate) unk_688: u32,
+        pub(crate) max_pstate_scaled_2: u32,
+        pub(crate) pad_690: u32,
+        pub(crate) unk_694: u32,
+        pub(crate) max_power_2: u32,
+        pub(crate) pad_69c: Pad<0x18>,
+        pub(crate) unk_6b4: u32,
+
+        pub(crate) unk_6b8_0: Array<0x10, u8>,
+
+        pub(crate) max_pstate_scaled_3: u32,
+        pub(crate) unk_6bc: u32,
+        pub(crate) pad_6c0: Pad<0x14>,
+        pub(crate) ppm_filter_tc_periods_x4: u32,
+        pub(crate) unk_6d8: u32,
+        pub(crate) pad_6dc: u32,
+        pub(crate) ppm_filter_a_neg: F32,
+        pub(crate) pad_6e4: u32,
+        pub(crate) ppm_filter_a: F32,
+        pub(crate) pad_6ec: u32,
+        pub(crate) ppm_ki_dt: F32,
+        pub(crate) pad_6f4: u32,
+        pub(crate) pwr_integral_min_clamp_2: u32,
+        pub(crate) unk_6fc: F32,
+        pub(crate) ppm_kp: F32,
+        pub(crate) pad_704: u32,
+        pub(crate) unk_708: u32,
+        pub(crate) pwr_min_duty_cycle: u32,
+        pub(crate) max_pstate_scaled_4: u32,
+        pub(crate) unk_714: u32,
+        pub(crate) pad_718: u32,
+        pub(crate) unk_71c: F32,
+        pub(crate) max_power_3: u32,
+        pub(crate) cur_power_mw_2: u32,
+        pub(crate) ppm_filter_tc_ms: u32,
+        pub(crate) unk_72c: u32,
+
+        pub(crate) ppm_filter_tc_clks: u32,
+
+        pub(crate) unk_730_4: u32,
+
+        pub(crate) unk_730_8: u32,
+
+        pub(crate) unk_730_c: u32,
+
+        pub(crate) unk_730: F32,
+        pub(crate) unk_734: u32,
+        pub(crate) unk_738: u32,
+        pub(crate) unk_73c: u32,
+        pub(crate) unk_740: u32,
+        pub(crate) unk_744: u32,
+        pub(crate) unk_748: Array<0x4, F32>,
+        pub(crate) unk_758: u32,
+        pub(crate) perf_tgt_utilization: u32,
+        pub(crate) pad_760: u32,
+        pub(crate) perf_boost_min_util: u32,
+        pub(crate) perf_boost_ce_step: u32,
+        pub(crate) perf_reset_iters: u32,
+        pub(crate) pad_770: u32,
+        pub(crate) unk_774: u32,
+        pub(crate) unk_778: u32,
+        pub(crate) perf_filter_drop_threshold: u32,
+        pub(crate) perf_filter_a_neg: F32,
+        pub(crate) perf_filter_a2_neg: F32,
+        pub(crate) perf_filter_a: F32,
+        pub(crate) perf_filter_a2: F32,
+        pub(crate) perf_ki: F32,
+        pub(crate) perf_ki2: F32,
+        pub(crate) perf_integral_min_clamp: F32,
+        pub(crate) unk_79c: F32,
+        pub(crate) perf_kp: F32,
+        pub(crate) perf_kp2: F32,
+        pub(crate) boost_state_unk_k: F32,
+        pub(crate) base_pstate_scaled_2: u32,
+        pub(crate) max_pstate_scaled_5: u32,
+        pub(crate) base_pstate_scaled_3: u32,
+        pub(crate) pad_7b8: u32,
+        pub(crate) perf_cur_utilization: F32,
+        pub(crate) perf_tgt_utilization_2: u32,
+        pub(crate) pad_7c4: Pad<0x18>,
+        pub(crate) unk_7dc: u32,
+
+        pub(crate) unk_7e0_0: Array<0x10, u8>,
+
+        pub(crate) base_pstate_scaled_4: u32,
+        pub(crate) pad_7e4: u32,
+        pub(crate) unk_7e8: Array<0x14, u8>,
+        pub(crate) unk_7fc: F32,
+        pub(crate) pwr_min_duty_cycle_2: F32,
+        pub(crate) max_pstate_scaled_6: F32,
+        pub(crate) max_freq_mhz: u32,
+        pub(crate) pad_80c: u32,
+        pub(crate) unk_810: u32,
+        pub(crate) pad_814: u32,
+        pub(crate) pwr_min_duty_cycle_3: u32,
+        pub(crate) unk_81c: u32,
+        pub(crate) pad_820: u32,
+        pub(crate) min_pstate_scaled_4: F32,
+        pub(crate) max_pstate_scaled_7: u32,
+        pub(crate) unk_82c: u32,
+        pub(crate) unk_alpha_neg: F32,
+        pub(crate) unk_alpha: F32,
+        pub(crate) unk_838: u32,
+        pub(crate) unk_83c: u32,
+        pub(crate) pad_840: Pad<0x2c>,
+        pub(crate) unk_86c: u32,
+        pub(crate) fast_die0_sensor_mask: U64,
+        pub(crate) fast_die0_release_temp_cc: u32,
+        pub(crate) unk_87c: i32,
+        pub(crate) unk_880: u32,
+        pub(crate) unk_884: u32,
+        pub(crate) pad_888: u32,
+        pub(crate) unk_88c: u32,
+        pub(crate) pad_890: u32,
+        pub(crate) unk_894: F32,
+        pub(crate) pad_898: u32,
+        pub(crate) fast_die0_ki_dt: F32,
+        pub(crate) pad_8a0: u32,
+        pub(crate) unk_8a4: u32,
+        pub(crate) unk_8a8: F32,
+        pub(crate) fast_die0_kp: F32,
+        pub(crate) pad_8b0: u32,
+        pub(crate) unk_8b4: u32,
+        pub(crate) pwr_min_duty_cycle_4: u32,
+        pub(crate) max_pstate_scaled_8: u32,
+        pub(crate) max_pstate_scaled_9: u32,
+        pub(crate) fast_die0_prop_tgt_delta: u32,
+        pub(crate) unk_8c8: u32,
+        pub(crate) unk_8cc: u32,
+        pub(crate) pad_8d0: Pad<0x14>,
+
+        pub(crate) unk_8e4_0: Array<0x10, u8>,
+
+        pub(crate) unk_8e4: u32,
+        pub(crate) unk_8e8: u32,
+        pub(crate) max_pstate_scaled_10: u32,
+        pub(crate) unk_8f0: u32,
+        pub(crate) unk_8f4: u32,
+        pub(crate) pad_8f8: u32,
+        pub(crate) pad_8fc: u32,
+        pub(crate) unk_900: Array<0x24, u8>,
+
+        pub(crate) unk_coef_a1: Array<8, Array<MAX_CORES_PER_CLUSTERG15V14_8_3, F32>>,
+        pub(crate) unk_coef_a2: Array<8, Array<MAX_CORES_PER_CLUSTERG15V14_8_3, F32>>,
+
+        pub(crate) pad_b24: Pad<0x70>,
+        pub(crate) max_pstate_scaled_11: u32,
+        pub(crate) freq_with_off: u32,
+        pub(crate) unk_b9c: u32,
+        pub(crate) unk_ba0: U64,
+        pub(crate) unk_ba8: U64,
+        pub(crate) unk_bb0: u32,
+        pub(crate) unk_bb4: u32,
+
+        pub(crate) pad_bb8_0: Pad<0x200>,
+        pub(crate) pad_bb8_200: Pad<0x8>,
+
+        pub(crate) pad_bb8: Pad<0x74>,
+        pub(crate) unk_c2c: u32,
+        pub(crate) power_zone_count: u32,
+        pub(crate) max_power_4: u32,
+        pub(crate) max_power_5: u32,
+        pub(crate) max_power_6: u32,
+        pub(crate) unk_c40: u32,
+        pub(crate) unk_c44: F32,
+        pub(crate) avg_power_target_filter_a_neg: F32,
+        pub(crate) avg_power_target_filter_a: F32,
+        pub(crate) avg_power_target_filter_tc_x4: u32,
+        pub(crate) avg_power_target_filter_tc_xperiod: u32,
+
+        pub(crate) avg_power_target_filter_tc_clks: u32,
+
+        pub(crate) unk_c58_4: u32,
+
+        pub(crate) power_zones: Array<5, PowerZoneG15V14_8_3>,
+        pub(crate) avg_power_filter_tc_periods_x4: u32,
+        pub(crate) unk_cfc: u32,
+        pub(crate) unk_d00: u32,
+        pub(crate) avg_power_filter_a_neg: F32,
+        pub(crate) unk_d08: u32,
+        pub(crate) avg_power_filter_a: F32,
+        pub(crate) unk_d10: u32,
+        pub(crate) avg_power_ki_dt: F32,
+        pub(crate) unk_d18: u32,
+        pub(crate) unk_d1c: u32,
+        pub(crate) unk_d20: F32,
+        pub(crate) avg_power_kp: F32,
+        pub(crate) unk_d28: u32,
+        pub(crate) unk_d2c: u32,
+        pub(crate) avg_power_min_duty_cycle: u32,
+        pub(crate) max_pstate_scaled_12: u32,
+        pub(crate) max_pstate_scaled_13: u32,
+        pub(crate) unk_d3c: u32,
+        pub(crate) max_power_7: F32,
+        pub(crate) max_power_8: u32,
+        pub(crate) unk_d48: u32,
+        pub(crate) avg_power_filter_tc_ms: u32,
+        pub(crate) unk_d50: u32,
+
+        pub(crate) avg_power_filter_tc_clks: u32,
+
+        pub(crate) unk_d54_4: Array<0xc, u8>,
+
+        pub(crate) unk_d54: Array<0x10, u8>,
+        pub(crate) max_pstate_scaled_14: u32,
+        pub(crate) unk_d68: Array<0x24, u8>,
+
+        pub(crate) t81xx_data: T81xxData,
+
+        pub(crate) unk_dd0: Array<0x40, u8>,
+
+        pub(crate) unk_e10_pad: Array<0x10, u8>,
+        pub(crate) unk_e10_0: HwDataA140Extra,
+
+        pub(crate) unk_e10: Array<0xc, u8>,
+
+        // G15 offsets match G13 13.5 up to ~0x1224 and are +0x10 from 0x1288
+        // (`fast_die0_sensor_mask_2`, G13 0x1278). TODO: exact insertion point between
+        // 0x1224 and 0x1288 is not known; placed as late as possible.
+        pub(crate) unk_1278_g15: Array<0x10, u8>,
+
+        pub(crate) fast_die0_sensor_mask_2: U64,
+
+        pub(crate) unk_e24: u32,
+        pub(crate) unk_e28: u32,
+        pub(crate) unk_e2c: Pad<0x1c>,
+        pub(crate) unk_coef_b1: Array<8, Array<MAX_CORES_PER_CLUSTERG15V14_8_3, F32>>,
+        pub(crate) unk_coef_b2: Array<8, Array<MAX_CORES_PER_CLUSTERG15V14_8_3, F32>>,
+
+        pub(crate) pad_1048: Pad<0x5e4>,
+
+        pub(crate) fast_die0_sensor_mask_alt: U64,
+
+        pub(crate) unk_163c: u32,
+
+        pub(crate) unk_1640: Array<0x2000, u8>,
+
+        pub(crate) unk_3640: u32,
+        pub(crate) unk_3644: u32,
+        pub(crate) hws1: HwDataShared1,
+
+        pub(crate) unk_hws2: Array<16, u16>,
+
+        pub(crate) hws2: HwDataShared2,
+        pub(crate) unk_3c00: u32,
+        pub(crate) unk_3c04: u32,
+        pub(crate) hws3: HwDataShared3,
+        pub(crate) unk_3c58: Array<0x3c, u8>,
+        pub(crate) unk_3c94: u32,
+        pub(crate) unk_3c98: U64,
+        pub(crate) unk_3ca0: U64,
+        pub(crate) unk_3ca8: U64,
+        pub(crate) unk_3cb0: U64,
+        pub(crate) ts_last_idle: U64,
+        pub(crate) ts_last_poweron: U64,
+        pub(crate) ts_last_poweroff: U64,
+        pub(crate) unk_3cd0: U64,
+        pub(crate) unk_3cd8: U64,
+
+        pub(crate) unk_3ce0_0: u32,
+
+        pub(crate) unk_3ce0: u32,
+        pub(crate) unk_3ce4: u32,
+        pub(crate) unk_3ce8: u32,
+        pub(crate) unk_3cec: u32,
+        pub(crate) unk_3cf0: u32,
+        pub(crate) core_leak_coef: Array<8, F32>,
+        pub(crate) sram_leak_coef: Array<8, F32>,
+
+        pub(crate) aux_leak_coef: AuxLeakCoef,
+        pub(crate) unk_3d34_0: Array<0x18, u8>,
+
+        pub(crate) unk_3d34: Array<0x38, u8>,
+
+        // G15 tail, 0x422c..0x4360.
+        pub(crate) unk_422c: Array<0x20, u8>,
+
+        /// System counter value at init. +0x424c
+        /// The runtime backend fills it; the manager leaves it 0, as validated.
+        pub(crate) init_timestamp: U64,
+        pub(crate) unk_4254: Array<0x40, u8>,
+
+        /// +0x4294/+0x4298
+        pub(crate) unk_4294: u32,
+        pub(crate) unk_4298: u32,
+        pub(crate) unk_429c: Array<0x8, u8>,
+
+        /// Three 8-entry per-cluster tables. +0x42a4
+        /// TODO: source values unknown; left 0.
+        pub(crate) cluster_tables: Array<3, Array<8, u32>>,
+
+        /// Eight words, source unknown. +0x4304..+0x4320
+        pub(crate) unk_4304: Array<8, u32>,
+        pub(crate) unk_4324: Array<0x1c, u8>,
+
+        /// Read-modify-written at init. +0x4340
+        pub(crate) unk_4340: U64,
+        pub(crate) unk_4348: Array<0x10, u8>,
+
+        /// GPU keep-alive mode (read by the host). +0x4358
+        pub(crate) gpu_keepalive_mode: u32,
+        pub(crate) unk_435c: u32,
+    }
+    default_zeroed!(HwDataAG15V14_8_3);
+    no_debug!(HwDataAG15V14_8_3);
+
+    #[derive(Debug, Default, Clone, Copy)]
+    #[repr(C)]
+    pub(crate) struct IOMapping {
+        pub(crate) phys_addr: U64,
+        pub(crate) virt_addr: U64,
+        pub(crate) total_size: u32,
+        pub(crate) element_size: u32,
+        pub(crate) readwrite: U64,
+    }
+
+    const IO_MAPPING_COUNTG15V14_8_3: usize = {
+        // 31 IOMapping slots (0x3e0 / 0x20)
+        {
+            0x1f
+        }
+    };
+
+    /// A G15 performance-state table: the highest entry, then per entry the frequency (MHz)
+    /// and the core and SRAM voltages (mV), eight voltage columns each.
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataBPStateTable {
+        pub(crate) max_pstate: u32,
+        pub(crate) frequencies: Array<0x10, u32>,
+        pub(crate) voltages: Array<0x10, [u32; 0x8]>,
+        pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataBAuxPStates {
+        pub(crate) cs_max_pstate: u32,
+        pub(crate) cs_frequencies: Array<0x10, u32>,
+        pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>,
+        pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>,
+        pub(crate) cs_unkpad: u32,
+        pub(crate) afr_max_pstate: u32,
+        pub(crate) afr_frequencies: Array<0x8, u32>,
+        pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>,
+        pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>,
+        pub(crate) afr_unkpad: u32,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct HwDataBG15V14_8_3 {
+
+        pub(crate) unk_8: U64,
+
+        pub(crate) unk_18: U64,
+        pub(crate) unk_20: U64,
+        pub(crate) unk_28: U64,
+        pub(crate) unk_30: U64,
+        pub(crate) timestamp_area_base: U64,
+
+        /// G15: a pointer. +0x30
+        /// TODO: what it points to; left 0.
+        pub(crate) unk_30_ptr: U64,
+
+        // G15: two tables of 32 CSC matrices (3x4 i16) at 0x38..0x637
+        pub(crate) yuv_matrices: Array<0x40, Array<3, Array<4, i16>>>,
+
+        pub(crate) pad_1c8: Pad<0x8>,
+        pub(crate) io_mappings: Array<IO_MAPPING_COUNTG15V14_8_3, IOMapping>,
+
+        pub(crate) sgx_sram_ptr: U64,
+
+        pub(crate) chip_id: u32,
+        pub(crate) unk_454: u32,
+        pub(crate) unk_458: u32,
+        pub(crate) unk_45c: u32,
+        pub(crate) unk_460: u32,
+        pub(crate) unk_464: u32,
+        pub(crate) unk_468: u32,
+        pub(crate) unk_46c: u32,
+        pub(crate) unk_470: u32,
+        pub(crate) unk_474: u32,
+        pub(crate) unk_478: u32,
+        pub(crate) unk_47c: u32,
+        pub(crate) unk_480: u32,
+        pub(crate) unk_484: u32,
+        pub(crate) unk_488: u32,
+        pub(crate) unk_48c: u32,
+        pub(crate) base_clock_khz: u32,
+        pub(crate) power_sample_period: u32,
+
+        // G15 0xa70..0xb3f. The G13 13.5 0x9b0..0xa73 group,
+        // with 12 bytes inserted at an unknown position; the words are named by their G15 offset.
+        // TODO: map these to the G13 names (`num_frags`, `unk_524` secure cache flush, ...)
+        // once their values are known.
+        pub(crate) unk_a70: u32,
+        pub(crate) unk_a74: u32,
+        pub(crate) unk_a78: u32,
+        pub(crate) unk_a7c: U64,
+        pub(crate) unk_a84: u32,
+        pub(crate) unk_a88: u32,
+        pub(crate) unk_a8c: u32,
+        pub(crate) unk_a90: u32,
+        pub(crate) unk_a94: u32,
+        pub(crate) unk_a98: u32,
+        pub(crate) unk_a9c: u32,
+        pub(crate) unk_aa0: u32,
+        pub(crate) unk_aa4: u32,
+        pub(crate) unk_aa8: u32,
+        pub(crate) unk_aac: u32,
+        pub(crate) unk_ab0: u32,
+        pub(crate) unk_ab4: u32,
+        pub(crate) unk_ab8: u32,
+        pub(crate) unk_abc: U64,
+        pub(crate) unk_ac4: u32,
+        pub(crate) unk_ac8: Array<0x4, u32>,
+        pub(crate) unk_ad8: u32,
+        pub(crate) unk_adc: u32,
+        pub(crate) unk_ae0: u32,
+        pub(crate) unk_ae4: u32,
+        pub(crate) unk_ae8: U64,
+        pub(crate) unk_af0: u32,
+        pub(crate) unk_af4: U64,
+        pub(crate) unk_afc: u32,
+        pub(crate) unk_b00: u32,
+        pub(crate) unk_b04: u32,
+        pub(crate) unk_b08: Array<0x4, u32>,
+        pub(crate) unk_b18: u32,
+        pub(crate) unk_b1c: u32,
+        pub(crate) unk_b20: u32,
+        pub(crate) unk_b24: U64,
+        pub(crate) unk_b2c: Array<0x14, u8>,
+
+        pub(crate) unk_554: u32,
+        pub(crate) uat_ttb_base: U64,
+        pub(crate) gpu_core_id: u32,
+        pub(crate) gpu_rev_id: u32,
+        pub(crate) num_cores: u32,
+        pub(crate) max_pstate: u32,
+
+        pub(crate) frequencies: Array<0x10, u32>,
+        pub(crate) voltages: Array<0x10, [u32; 0x8]>,
+        pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>,
+
+
+        /// G15: second 16-entry frequency table in MHz (meaning inferred). +0xf9c
+        pub(crate) frequencies_2: Array<0x10, u32>,
+
+        pub(crate) sram_k: Array<0x10, F32>,
+        pub(crate) unk_9f4: Array<0x10, u32>,
+        pub(crate) rel_max_powers: Array<0x10, u32>,
+        pub(crate) rel_boost_freqs: Array<0x10, u32>,
+
+        pub(crate) unk_arr_0: Array<32, u32>,
+
+        pub(crate) aux_ps: HwDataBAuxPStates,
+
+        // G15 0x134c..0x17eb: a third p-state table at 0x134c, then the
+        // words before `timer_offset`. G15 is +4 relative to G13 here (+0xd0 at `timer_offset`).
+
+        /// Third p-state table: the secondary (lowest frequency per voltage) table, with one
+        /// voltage column. +0x134c
+        pub(crate) aux_ps_3: HwDataBPStateTable,
+        pub(crate) aux_ps_3_pad: Array<0xc, u8>,
+        pub(crate) unk_179c: U64,
+        pub(crate) unk_17a4: Array<0x10, u8>,
+        pub(crate) unk_17b4: u32,
+        pub(crate) unk_17b8: u32,
+        pub(crate) unk_17bc: u32,
+
+        /// Unit enable bit masks. +0x17c0/+0x17c8
+        pub(crate) unit_mask_a: U64,
+        pub(crate) unit_mask_b: u32,
+        pub(crate) unk_17cc: u32,
+        pub(crate) unk_17d0: u32,
+        pub(crate) unk_17d4: Array<0x10, u8>,
+        pub(crate) unk_17e4: u32,
+        pub(crate) unk_17e8: u32,
+
+        pub(crate) timer_offset: U64,
+
+        // G15 0x17f4..0x1867.
+        pub(crate) unk_17f4: U64,
+        pub(crate) unk_17fc: u32,
+        pub(crate) unk_1800: u32,
+        pub(crate) unk_1804: u32,
+        pub(crate) unk_1808: u32,
+        pub(crate) unk_180c: u32,
+
+        /// +0x1810 and +0x1814 are both 1 (written as one 64-bit store); the firmware copies them
+        /// to separate globals.
+        pub(crate) unk_1810: u32,
+        pub(crate) unk_1814: u32,
+        pub(crate) unk_1818: u32,
+        pub(crate) unk_181c: u32,
+
+        /// 48 bytes of 0xff ("all units enabled" masks?). +0x1820
+        pub(crate) all_ones_masks: Array<0x30, u8>,
+
+        /// FSTP override enable. +0x1850
+        pub(crate) fstp_override: u32,
+        pub(crate) unk_1854: U64,
+        pub(crate) unk_185c: u32,
+        pub(crate) unk_1860: u32,
+        pub(crate) unk_1864: u32,
+    }
+    default_zeroed!(HwDataBG15V14_8_3);
+
+    #[derive(Debug)]
+    #[repr(C, packed)]
+    pub(crate) struct GpuStatsVtx {
+        // This changes all the time and we don't use it, let's just make it a big buffer
+        pub(crate) opaque: Array<0x3000, u8>,
+    }
+    default_zeroed!(GpuStatsVtx);
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct GpuStatsFragG15V14_8_3 {
+        // G15: the two -1 fields are at E2+0xc18/+0xc30
+        pub(crate) unk1_0: Array<0xb10, u8>,
+        pub(crate) unk1: Array<0x100, u8>,
+        pub(crate) cur_stamp_id: i32,
+        pub(crate) unk2: Array<0x14, u8>,
+        pub(crate) unk_id: i32,
+        // G15 E2 is 0x1248 bytes in total
+        pub(crate) unk3: Array<0x614, u8>,
+    }
+
+    impl Default for GpuStatsFragG15V14_8_3 {
+        fn default() -> Self {
+            Self {
+                unk1_0: Default::default(),
+                unk1: Default::default(),
+                cur_stamp_id: -1,
+                unk2: Default::default(),
+                unk_id: -1,
+                unk3: Default::default(),
+            }
+        }
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct GpuGlobalStatsVtx {
+        pub(crate) total_cmds: u32,
+        pub(crate) stats: GpuStatsVtx,
+    }
+    default_zeroed!(GpuGlobalStatsVtx);
+
+    #[derive(Debug, Default)]
+    #[repr(C)]
+    pub(crate) struct GpuGlobalStatsFragG15V14_8_3 {
+        pub(crate) total_cmds: u32,
+        pub(crate) unk_4: u32,
+        pub(crate) stats: GpuStatsFragG15V14_8_3,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct GpuStatsComp {
+        // This changes all the time and we don't use it, let's just make it a big buffer
+        pub(crate) opaque: Array<0x3000, u8>,
+    }
+    default_zeroed!(GpuStatsComp);
+
+    // Not part of the 0x490-byte 14.x RuntimePointers, so the G15V14_8_3
+    // instantiation is unused.
+    #[derive(Debug)]
+    #[repr(C)]
+    #[allow(dead_code)]
+    pub(crate) struct RuntimeScratchG15V14_8_3 {
+        pub(crate) unk_280: Array<0x6800, u8>,
+        pub(crate) unk_6a80: u32,
+        pub(crate) gpu_idle: u32,
+        pub(crate) unkpad_6a88: Pad<0x14>,
+        pub(crate) unk_6a9c: u32,
+        pub(crate) unk_ctr0: u32,
+        pub(crate) unk_ctr1: u32,
+        pub(crate) unk_6aa8: u32,
+        pub(crate) unk_6aac: u32,
+        pub(crate) unk_ctr2: u32,
+        pub(crate) unk_6ab4: u32,
+        pub(crate) unk_6ab8: u32,
+        pub(crate) unk_6abc: u32,
+        pub(crate) unk_6ac0: u32,
+        pub(crate) unk_6ac4: u32,
+        pub(crate) unk_ctr3: u32,
+        pub(crate) unk_6acc: u32,
+        pub(crate) unk_6ad0: u32,
+        pub(crate) unk_6ad4: u32,
+        pub(crate) unk_6ad8: u32,
+        pub(crate) unk_6adc: u32,
+        pub(crate) unk_6ae0: u32,
+        pub(crate) unk_6ae4: u32,
+        pub(crate) unk_6ae8: u32,
+        pub(crate) unk_6aec: u32,
+        pub(crate) unk_6af0: u32,
+        pub(crate) unk_ctr4: u32,
+        pub(crate) unk_ctr5: u32,
+        pub(crate) unk_6afc: u32,
+        pub(crate) pad_6b00: Pad<0x38>,
+
+        pub(crate) pad_6b00_extra: Array<0x4800, u8>,
+
+        pub(crate) unk_6b38: u32,
+        pub(crate) pad_6b3c: Pad<0x84>,
+    }
+    default_zeroed!(RuntimeScratchG15V14_8_3);
+
+    #[repr(C)]
+    pub(crate) struct RuntimePointersG15V14_8_3<'a> {
+        // G15 / 14.8.3: 0x490 bytes, packed (HwDataA pointer at +0x441). GpuPointer/U64 are
+        // packed(1), so repr(C) gives the firmware offsets.
+
+        /// +0x000
+        pub(crate) hwdata_b: GpuPointer<'a, super::HwDataBG15V14_8_3>,
+
+        /// Hardware-erratum (BRN) table. +0x008
+        pub(crate) brn_table: GpuPointer<'a, &'a [u8]>,
+
+        /// E7 (0x88 bytes, zeroed). +0x010
+        pub(crate) unkptr_e7: GpuPointer<'a, &'a [u8]>,
+
+        pub(crate) pipes: Array<4, PipeChannelsG15V14_8_3>,
+
+        /// +0x198
+        pub(crate) device_control: ScratchRingDesc,
+        pub(crate) event: ChannelRing<channels::ChannelState, channels::RawEventMsg>,
+        pub(crate) fw_log: ChannelRing<channels::FwLogChannelState, channels::RawFwLogMsg>,
+        pub(crate) ktrace: ChannelRing<channels::ChannelState, channels::RawKTraceMsg>,
+        // The G15 stats record has the 13.x size (STATS_SZ for V >= V13_0B4); 7.2 generates
+        // only the 13.x instances.
+        pub(crate) stats: ChannelRing<channels::ChannelState, channels::RawStatsMsgG13V13_5>,
+
+
+        /// FWLog payload buffer (0x600 x 0xd8). +0x1f8
+        pub(crate) fwlog_buf: Option<GpuWeakPointer<[channels::RawFwLogPayloadMsg]>>,
+        pub(crate) __pad_200: Pad<0x30>,
+
+        /// 0 at init. +0x230
+        pub(crate) unk_230: u32,
+        pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>,
+        pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFragG15V14_8_3>,
+        pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>,
+
+
+        /// E5 (0x60 bytes, zeroed; FW status word at +0). +0x24c
+        pub(crate) unkptr_e5: GpuPointer<'a, &'a [u8]>,
+        pub(crate) __pad_254: Pad<0x54>,
+
+        /// Optional pointer, left 0. +0x2a8
+        pub(crate) unkptr_2a8: U64,
+
+        /// PB descriptor table GPU/FW VA. +0x2b0/+0x2b8
+        pub(crate) buffer_mgr_ctl_gpu_addr: U64,
+        pub(crate) buffer_mgr_ctl_fw_addr: U64,
+
+        /// UMA page-pool descriptor table GPU/FW VA. +0x2c0/+0x2c8
+        pub(crate) uma_pool_desc_gpu_addr: U64,
+        pub(crate) uma_pool_desc_fw_addr: U64,
+
+        /// Configuration words, left 0. +0x2d0/+0x2d4
+        pub(crate) unk_2d0: u32,
+        pub(crate) unk_2d4: u32,
+        pub(crate) __pad_2d8: Pad<0xd8>,
+
+        /// 0xff at init (context-switch histogram control). +0x3b0
+        pub(crate) cswitch_hist_ctl: u8,
+        pub(crate) unk_3b1: Array<0x90, u8>,
+
+        // G15: +0x441 (unaligned)
+        pub(crate) hwdata_a: GpuPointer<'a, super::HwDataAG15V14_8_3>,
+
+
+        /// 16 configuration bytes, left 0. +0x449
+        pub(crate) unk_449: Array<0x10, u8>,
+        pub(crate) __pad_459: Pad<0x14>,
+
+        /// Zero at init. +0x46d
+        pub(crate) unk_46d: Array<0x20, u8>,
+        pub(crate) __pad_48d: Pad<0x3>,
+    }
+    no_debug!(RuntimePointersG15V14_8_3<'_>);
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct PendingStamp {
+        pub(crate) info: AtomicU32,
+        pub(crate) wait_value: AtomicU32,
+    }
+    default_zeroed!(PendingStamp);
+
+    #[derive(Debug, Clone, Copy)]
+    #[repr(C, packed)]
+    pub(crate) struct FaultInfo {
+        pub(crate) unk_0: u32,
+        pub(crate) unk_4: u32,
+        pub(crate) queue_uuid: u32,
+        pub(crate) unk_c: u32,
+        pub(crate) unk_10: u32,
+        pub(crate) unk_14: u32,
+    }
+    default_zeroed!(FaultInfo);
+
+    #[derive(Debug, Clone, Copy)]
+    #[repr(C)]
+    pub(crate) struct PowerZoneGlobal {
+        pub(crate) target: u32,
+        pub(crate) target_off: u32,
+        pub(crate) filter_tc: u32,
+    }
+    default_zeroed!(PowerZoneGlobal);
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct GlobalsG15V14_8_3 {
+        // G15 / 14.8.3 layout: 0xe00 bytes, cached, packed.
+        // It re-packs the G13 field groups; the large G13 arrays are gone.
+
+        /// Timer context-switch mode, 0 or 7. +0x0
+        pub(crate) timer_cswitch: u32,
+        pub(crate) unk_4: u32,
+
+        /// +0x8
+        pub(crate) dm_pause_mode: u32,
+        pub(crate) unk_c: u32,
+
+        /// +0x10
+        pub(crate) dm_pause_timer: u32,
+        pub(crate) unk_14: u32,
+
+        /// +0x18
+        pub(crate) disable_cswitch: u32,
+
+        /// Configuration word. +0x1c
+        pub(crate) unk_1c: u32,
+
+        /// +0x20/+0x24
+        pub(crate) relaxed_cl_kill_timeout: u32,
+        pub(crate) frg_task_timeout: u32,
+
+        /// G13 `unk_24`. +0x28
+        pub(crate) unk_28: u32,
+
+        /// Debug flags bit 0 (G13 `debug`). +0x2c
+        pub(crate) debug: u32,
+
+        /// +0x30
+        pub(crate) smart_idle_off_enable: u32,
+        pub(crate) unk_34: u32,
+
+        /// Constant 0x78. +0x38
+        pub(crate) unk_38: u32,
+
+        /// +0x3c/+0x40
+        pub(crate) cpms_window_size: u32,
+        pub(crate) cpms_tfca_size: u32,
+        pub(crate) unk_44: u32,
+
+        /// Kick channel QoS pair. +0x48/+0x4c
+        pub(crate) kick_channel_qos_b: u32,
+        pub(crate) kick_channel_qos_a: u32,
+
+        /// G13 `unk_54`/`unk_56`/`unk_58` moved -0x14. +0x50..+0x54 (inferred)
+        pub(crate) unk_50: u16,
+        pub(crate) unk_52: u16,
+        pub(crate) unk_54: u16,
+
+        /// G13 `unk_5a`/`unk_5e`/`unk_62` moved -0x14 (unaligned). +0x56..+0x5e
+        pub(crate) unk_56: U32,
+        pub(crate) unk_5a: U32,
+        pub(crate) cswitch_timer_multiplier: U32,
+
+        /// +0x62 (G13 `unk_66`, tentative)
+        pub(crate) cdm_cs_mode_change: U32,
+        pub(crate) unk_66: Array<0xa, u8>,
+
+        /// Command submission enable; must be non-zero to submit work. +0x70
+        pub(crate) command_submission_enabled: u32,
+
+        /// Pending-submission counter (G13 `pending_submissions`). +0x74
+        pub(crate) pending_submissions: AtomicU32,
+
+        /// +0x78..+0x80
+        pub(crate) power_if_target: Array<3, u32>,
+
+        /// +0x84..+0x8c
+        pub(crate) controller_override_0: u32,
+        pub(crate) controller_override_1: u32,
+        pub(crate) perf_state_cap: u32,
+
+        /// Perf-controller block. +0x90..+0xc7
+        pub(crate) perf_controller: Array<0x38, u8>,
+
+        /// +0xc8/+0xcc
+        pub(crate) deadline_control_effort: u32,
+        pub(crate) deadline_controller_override: u32,
+        pub(crate) unk_d0: Array<0xa0, u8>,
+
+        /// DPE (dynamic power estimation) block. +0x170..+0x69f
+        pub(crate) dpe: Array<0x530, u8>,
+
+        /// +0x6a0
+        pub(crate) dpe_sl_config: U64,
+        pub(crate) unk_6a8: Array<0x100, u8>,
+
+        /// Smart idle-off config. +0x7a8
+        pub(crate) smart_idle_off_cfg: Array<10, u32>,
+
+        /// +0x7d0..+0x7dc
+        pub(crate) ut_engagement: u32,
+        pub(crate) clvr_engagement: u32,
+        pub(crate) keepalive_perf_threshold_rd: u32,
+        pub(crate) keepalive_off_threshold_rd: u32,
+        pub(crate) unk_7e0: u32,
+
+        /// Register overrides. +0x7e4
+        pub(crate) reg_overrides: Array<16, RegisterOverride>,
+
+        /// +0x964
+        pub(crate) reg_override_count: u32,
+        pub(crate) unk_968: u32,
+
+        /// Display power-management params. +0x96c/+0x974
+        pub(crate) dpm_param_a: U64,
+        pub(crate) dpm_param_b: U64,
+
+        /// +0x97c..+0x984
+        pub(crate) progress_check_interval_3d: u32,
+        pub(crate) progress_check_interval_ta: u32,
+        pub(crate) progress_check_interval_cl: u32,
+
+        /// +0x988..+0x99c: the G13 `unk_1102c_0`..`unk_1102c` group (inferred)
+        pub(crate) unk_1102c_0: u32,
+        pub(crate) unk_1102c_4: u32,
+        pub(crate) unk_1102c_8: u32,
+        pub(crate) unk_1102c_c: u32,
+        pub(crate) unk_1102c_10: u32,
+        pub(crate) unk_1102c: u32,
+
+        /// +0x9a0..+0x9a8
+        pub(crate) idle_off_delay_ms: AtomicU32,
+        pub(crate) fender_idle_off_delay_ms: u32,
+        pub(crate) fw_early_wake_timeout_ms: u32,
+
+        /// +0x9ac
+        pub(crate) gvdm_timer_interval: u32,
+
+        /// +0x9b0/+0x9b4
+        pub(crate) cl_context_switch_timeout_ms: u32,
+        pub(crate) cl_kill_timeout_ms: u32,
+        pub(crate) unk_9b8: u32,
+
+        /// +0x9bc
+        pub(crate) cdm_backoff_timeout: u8,
+        pub(crate) unk_9bd: Array<0x4, u8>,
+
+        /// +0x9c1/+0x9c3
+        pub(crate) fwutil_default_fab_pstate: Array<0x2, u8>,
+        pub(crate) fwutil_timer_period: u8,
+        pub(crate) unk_9c4: Array<0x19, u8>,
+
+        /// Unaligned. +0x9dd
+        pub(crate) mtr_sensor_ptd_override_mask: U64,
+        pub(crate) unk_9e5: Array<0x400, u8>,
+
+        /// Read by the host when it creates command queues. +0xde5
+        pub(crate) unk_de5: U32,
+
+        /// Keep-alive overrides/thresholds. +0xde9..+0xdf5
+        pub(crate) gpu_keepalive_override: U32,
+        pub(crate) gfxc_keepalive_override: U32,
+        pub(crate) keepalive_perf_threshold: U32,
+        pub(crate) keepalive_off_threshold: U32,
+
+        /// Soft-fault settings (possibly G13 `fault_control`). +0xdf9
+        pub(crate) soft_fault_settings: U32,
+        pub(crate) unk_dfd: Array<0x3, u8>,
+
+        /// The G13 pending-stamp table is not in the 0xe00-byte G15 Globals. This
+        /// zero-length field keeps `GpuManager::mark_pending_events()` compiling as a no-op.
+        /// TODO: find the G15 pending-stamp table (possibly in P0 0x0008..0x4017).
+        pub(crate) pending_stamps: Array<0, PendingStamp>,
+    }
+    default_zeroed!(GlobalsG15V14_8_3);
+
+    #[derive(Debug, Default, Clone, Copy)]
+    #[repr(C, packed)]
+    pub(crate) struct UatLevelInfo {
+        pub(crate) unk_3: u8,
+        pub(crate) unk_1: u8,
+        pub(crate) unk_2: u8,
+        pub(crate) index_shift: u8,
+        pub(crate) num_entries: u16,
+        pub(crate) unk_4: u16,
+        pub(crate) unk_8: U64,
+        pub(crate) unk_10: U64,
+        pub(crate) index_mask: U64,
+    }
+
+    #[derive(Debug)]
+    #[repr(C)]
+    pub(crate) struct InitDataG15V14_8_3<'a> {
+        pub(crate) ver_info: Array<0x4, u16>,
+
+        pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>,
+        pub(crate) unk_8: u32,
+        pub(crate) unk_c: u32,
+        pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointersG15V14_8_3>,
+        pub(crate) globals: GpuPointer<'a, super::GlobalsG15V14_8_3>,
+
+        /// G15: no FwStatus pointer; u32 0 then u32 1 (possibly a host-mapped FW allocations
+        /// flag). +0x28/+0x2c
+        pub(crate) unk_28: u32,
+        pub(crate) unk_2c: u32,
+        // 0x30..0x97 on G15 is a 0x68-byte copy of the UAT pmap descriptor (+0x18..+0x7f). Its
+        // first 0x64 bytes have the G13 layout (3 levels, 0x20-byte records).
+        pub(crate) uat_page_size: u16,
+        pub(crate) uat_page_bits: u8,
+        pub(crate) uat_num_levels: u8,
+        pub(crate) uat_level_info: Array<0x3, UatLevelInfo>,
+
+
+        /// G15: last 4 bytes of the pmap descriptor copy (descriptor +0x7c).
+        /// TODO: meaning unknown; left 0.
+        pub(crate) unk_94: u32,
+        pub(crate) __pad_98: Pad<0x10>,
+
+        /// P2 debug block (0x20). +0xa8
+        pub(crate) debug_block: GpuPointer<'a, super::DebugBlock>,
+
+        /// P0 status block (0xc3d0). +0xb0
+        pub(crate) status_block: GpuPointer<'a, super::StatusBlock>,
+
+        /// P1 power-controller block (0x238). +0xb8
+        pub(crate) power_ctl_block: GpuPointer<'a, super::PowerCtlBlock>,
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct ChannelRing<T: GpuStruct + Debug + Default, U: Copy>
+where
+    for<'a> <T as GpuStruct>::Raw<'a>: Debug,
+{
+    pub(crate) state: GpuObject<T>,
+    pub(crate) ring: GpuArray<U>,
+}
+
+impl<T: GpuStruct + Debug + Default, U: Copy> ChannelRing<T, U>
+where
+    for<'a> <T as GpuStruct>::Raw<'a>: Debug,
+{
+    pub(crate) fn to_raw(&self) -> raw::ChannelRing<T, U> {
+        raw::ChannelRing {
+            state: Some(self.state.weak_pointer()),
+            ring: Some(self.ring.weak_pointer()),
+        }
+    }
+}
+
+trivial_gpustruct!(FwStatus);
+trivial_gpustruct!(StatusBlock);
+trivial_gpustruct!(PowerCtlBlock);
+trivial_gpustruct!(DebugBlock);
+trivial_gpustruct!(GpuGlobalStatsVtx);
+trivial_gpustruct!(GpuGlobalStatsFragG15V14_8_3);
+trivial_gpustruct!(GpuStatsComp);
+
+trivial_gpustruct!(HwDataAG15V14_8_3);
+
+trivial_gpustruct!(HwDataBG15V14_8_3);
+
+#[derive(Debug)]
+pub(crate) struct StatsG15V14_8_3 {
+    pub(crate) vtx: GpuObject<GpuGlobalStatsVtx>,
+    pub(crate) frag: GpuObject<GpuGlobalStatsFragG15V14_8_3>,
+    pub(crate) comp: GpuObject<GpuStatsComp>,
+}
+
+#[derive(Debug)]
+pub(crate) struct RuntimePointersG15V14_8_3 {
+    pub(crate) stats: StatsG15V14_8_3,
+
+    pub(crate) hwdata_a: GpuObject<HwDataAG15V14_8_3>,
+    pub(crate) hwdata_b: GpuObject<HwDataBG15V14_8_3>,
+
+    // G15 group-A regions referenced from RuntimePointers
+    pub(crate) brn_table: GpuArray<u8>,
+    pub(crate) unkptr_e7: GpuArray<u8>,
+    pub(crate) unkptr_e5: GpuArray<u8>,
+    pub(crate) uma_pool_desc: GpuArray<u8>,
+
+    pub(crate) buffer_mgr_ctl: gem::ObjectRef,
+    pub(crate) buffer_mgr_ctl_low_mapping: Option<mmu::KernelMapping>,
+    pub(crate) buffer_mgr_ctl_high_mapping: Option<mmu::KernelMapping>,
+}
+
+impl GpuStruct for RuntimePointersG15V14_8_3 {
+    type Raw<'a> = raw::RuntimePointersG15V14_8_3<'a>;
+}
+
+trivial_gpustruct!(GlobalsG15V14_8_3);
+
+#[derive(Debug)]
+pub(crate) struct InitDataG15V14_8_3 {
+    pub(crate) unk_buf: GpuArray<u8>,
+    pub(crate) runtime_pointers: GpuObject<RuntimePointersG15V14_8_3>,
+    pub(crate) globals: GpuObject<GlobalsG15V14_8_3>,
+
+    /// G15: the P0 status block, which embeds the FwStatus FWCtl ring and halt flags. It keeps
+    /// the `fw_status` name so the FwStatus users see the same fields.
+    pub(crate) fw_status: GpuObject<StatusBlock>,
+    pub(crate) power_ctl_block: GpuObject<PowerCtlBlock>,
+    pub(crate) debug_block: GpuObject<DebugBlock>,
+}
+
+impl GpuStruct for InitDataG15V14_8_3 {
+    type Raw<'a> = raw::InitDataG15V14_8_3<'a>;
+}
+
+// G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects.
+mod g15_layout {
+    use crate::fw::channels;
+    use super::raw::*;
+    use core::mem::{offset_of, size_of};
+    use kernel::static_assert;
+
+    // InitData, 0xc0 bytes used
+    static_assert!(size_of::<InitDataG15V14_8_3<'static>>() == 0xc0);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_buf) == 0x08);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, runtime_pointers) == 0x18);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, globals) == 0x20);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_2c) == 0x2c);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, uat_page_size) == 0x30);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, debug_block) == 0xa8);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, status_block) == 0xb0);
+    static_assert!(offset_of!(InitDataG15V14_8_3<'static>, power_ctl_block) == 0xb8);
+
+    // host->FW ring descriptor
+    static_assert!(size_of::<ScratchRingDesc>() == 0x20);
+    static_assert!(size_of::<PipeChannelsG15V14_8_3>() == 0x60);
+
+    // RuntimePointers: 0x490 bytes, packed
+    static_assert!(size_of::<RuntimePointersG15V14_8_3<'static>>() == 0x490);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_b) == 0x000);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, brn_table) == 0x008);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e7) == 0x010);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, pipes) == 0x018);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, device_control) == 0x198);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, event) == 0x1b8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fw_log) == 0x1c8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, ktrace) == 0x1d8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats) == 0x1e8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fwlog_buf) == 0x1f8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_230) == 0x230);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_vtx) == 0x234);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_frag) == 0x23c);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_comp) == 0x244);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e5) == 0x24c);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_2a8) == 0x2a8);
+    static_assert!(
+        offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_gpu_addr) == 0x2b0
+    );
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_fw_addr) == 0x2b8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_gpu_addr) == 0x2c0);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_fw_addr) == 0x2c8);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_2d0) == 0x2d0);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, cswitch_hist_ctl) == 0x3b0);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_a) == 0x441);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_449) == 0x449);
+    static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_46d) == 0x46d);
+
+    // Globals: 0xe00 bytes, packed
+    static_assert!(size_of::<GlobalsG15V14_8_3>() == 0xe00);
+    static_assert!(size_of::<RegisterOverride>() == 0x18);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3::m3_globals_debug::OFFSET);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, cswitch_timer_multiplier) == 0x5e);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, command_submission_enabled) == 0x70);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, pending_submissions) == 0x74);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, perf_controller) == 0x90);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, deadline_control_effort) == 0xc8);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, dpe) == 0x170);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, dpe_sl_config) == 0x6a0);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, smart_idle_off_cfg) == 0x7a8);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, ut_engagement) == 0x7d0);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, reg_overrides) == 0x7e4);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, reg_override_count) == 0x964);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, dpm_param_a) == 0x96c);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, progress_check_interval_3d) == 0x97c);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_1102c_0) == 0x988);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, idle_off_delay_ms) == 0x9a0);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, fender_idle_off_delay_ms) == 0x9a4);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, fw_early_wake_timeout_ms) == 0x9a8);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, cl_context_switch_timeout_ms) == 0x9b0);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, cdm_backoff_timeout) == 0x9bc);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, fwutil_default_fab_pstate) == 0x9c1);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, mtr_sensor_ptd_override_mask) == 0x9dd);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, unk_de5) == 0xde5);
+    static_assert!(offset_of!(GlobalsG15V14_8_3, soft_fault_settings) == 0xdf9);
+
+    // P0 status block
+    static_assert!(size_of::<StatusBlock>() == 0xc3d0);
+    static_assert!(offset_of!(StatusBlock, fw_recovery_info_a) == 0x40a0);
+    static_assert!(offset_of!(StatusBlock, recovery_packet_info) == 0x44a0);
+    static_assert!(offset_of!(StatusBlock, unk_44c0) == 0x44c0);
+    static_assert!(offset_of!(StatusBlock, unk_4558) == 0x4558);
+    static_assert!(offset_of!(StatusBlock, fwctl_channel) == 0x4568);
+    static_assert!(offset_of!(StatusBlock, flags) == 0x4580);
+    static_assert!(offset_of!(StatusBlock, flags.halted) == 0x4590);
+    static_assert!(offset_of!(StatusBlock, flags.resume) == 0x45a0);
+    static_assert!(offset_of!(StatusBlock, flags.unk_40) == 0x45b0);
+    static_assert!(offset_of!(StatusBlock, flags.sched_callbacks) == 0x45c4);
+    static_assert!(offset_of!(StatusBlock, qos_mode) == 0xc3c8);
+
+    // P1 power-controller block
+    static_assert!(size_of::<PowerCtlBlock>() == 0x238);
+    static_assert!(offset_of!(PowerCtlBlock, power_zone_count) == 0x0c);
+    static_assert!(offset_of!(PowerCtlBlock, power_zones) == 0x24);
+    static_assert!(offset_of!(PowerCtlBlock, clvr_loop_override_a) == 0x60);
+    static_assert!(offset_of!(PowerCtlBlock, lifetime_servo_temp_a) == 0xa4);
+    static_assert!(offset_of!(PowerCtlBlock, fast_die0_release_temp) == 0xac);
+    static_assert!(offset_of!(PowerCtlBlock, ppm_words) == 0xc8);
+    static_assert!(offset_of!(PowerCtlBlock, unk_dc) == 0xdc);
+    static_assert!(offset_of!(PowerCtlBlock, se_controller_override) == 0xe8);
+    static_assert!(offset_of!(PowerCtlBlock, se_engagement) == 0xec);
+    static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_a) == 0x158);
+    static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_b) == 0x1a8);
+    static_assert!(offset_of!(PowerCtlBlock, clpc_shared_a) == 0x1b4);
+    static_assert!(offset_of!(PowerCtlBlock, perf_ctrl_override_active) == 0x1d0);
+    static_assert!(offset_of!(PowerCtlBlock, consistent_perf_state) == 0x1d1);
+    static_assert!(offset_of!(PowerCtlBlock, accumulated_energy) == 0x1d8);
+    static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_a) == 0x1e8);
+    static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_b) == 0x218);
+    static_assert!(offset_of!(PowerCtlBlock, submit_in_progress_gate) == 0x230);
+
+    // P2 debug block
+    static_assert!(size_of::<DebugBlock>() == 0x20);
+    static_assert!(offset_of!(DebugBlock, system_sleep_in_progress) == 0x08);
+    static_assert!(offset_of!(DebugBlock, fw_word_10) == 0x10);
+    static_assert!(offset_of!(DebugBlock, init_state) == 0x14);
+
+    // InitSeq record
+    static_assert!(size_of::<InitSeqRecord>() == 0x18);
+    static_assert!(offset_of!(InitSeqRecord, kind) == 0x10);
+
+    // HwDataB: 0x1868 bytes
+    static_assert!(size_of::<HwDataBG15V14_8_3>() == 0x1868);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_30) == 0x20);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, timestamp_area_base) == 0x28);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, yuv_matrices) == 0x38);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, pad_1c8) == 0x638);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, io_mappings) == 0x640);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, sgx_sram_ptr) == 0xa20);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, chip_id) == 0xa28);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, base_clock_khz) == 0xa68);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, power_sample_period) == 0xa6c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_a9c) == 0xa9c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_ac8) == 0xac8);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_b24) == 0xb24);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_554) == 0xb40);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, uat_ttb_base) == 0xb44);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, gpu_core_id) == 0xb4c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, num_cores) == 0xb54);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, max_pstate) == 0xb58);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies) == 0xb5c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, voltages) == 0xb9c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, voltages_sram) == 0xd9c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies_2) == 0xf9c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, sram_k) == 0xfdc);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_9f4) == 0x101c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_arr_0) == 0x10dc);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps) == 0x115c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps_3) == 0x134c);
+    static_assert!(size_of::<HwDataBPStateTable>() == 0x444);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_179c) == 0x179c);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unit_mask_a) == 0x17c0);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_17e8) == 0x17e8);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, timer_offset) == 0x17ec);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, all_ones_masks) == 0x1820);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, fstp_override) == 0x1850);
+    static_assert!(offset_of!(HwDataBG15V14_8_3, unk_1860) == 0x1860);
+    static_assert!(size_of::<IOMapping>() == 0x20);
+
+    // HwDataA: 0x4360 bytes, G13 13.5 layout, +0x10 from 0x1288
+    static_assert!(size_of::<HwDataAG15V14_8_3>() == 0x4360);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, clocks_per_period) == 0x4);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_status) == 0x10);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, actual_pstate) == 0x2c);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled) == 0x54);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, sram_k) == 0x80);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_64c) == 0x658);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_filter_a_neg) == 0x664);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, max_power_2) == 0x6a4);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization) == 0x788);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, perf_filter_drop_threshold) == 0x7a8);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization_2) == 0x7ec);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled_4) == 0x860);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask) == 0x8ac);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_release_temp_cc) == 0x8b4);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_ki_dt) == 0x8d8);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_kp) == 0x8e8);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_prop_tgt_delta) == 0x900);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_10) == 0x938);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_coef_a1) == 0x970);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_11) == 0xbe0);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_c2c) == 0xe80);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, power_zone_count) == 0xe84);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_target_filter_a_neg) == 0xe9c);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_periods_x4) == 0xf7c);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_kp) == 0xfa8);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_ms) == 0xfd0);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_14) == 0xff8);
+    // The builder relies on this base for the G15 +0x10f4/+0x11d0 stores
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e10_0) == 0x10b4);
+    static_assert!(size_of::<HwDataA140Extra>() == 0x1b8);
+    static_assert!(offset_of!(HwDataA140Extra, unk_40) == 0x40);
+    static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_a_neg) == 0x4c);
+    static_assert!(offset_of!(HwDataA140Extra, gpu_se_kp) == 0x6c);
+    static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_1) == 0x7c);
+    static_assert!(offset_of!(HwDataA140Extra, se_target) == 0x8c);
+    static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_time_constant_clks) == 0x9c);
+    static_assert!(offset_of!(HwDataA140Extra, unk_b0) == 0xb0);
+    static_assert!(offset_of!(HwDataA140Extra, unk_d8) == 0xd8);
+    static_assert!(offset_of!(HwDataA140Extra, gpu_se_reset_criteria) == 0xf0);
+    static_assert!(offset_of!(HwDataA140Extra, unk_11c) == 0x11c);
+    static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_2) == 0x130);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_2) == 0x1288);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e28) == 0x1294);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_alt) == 0x1a98);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_3640) == 0x3aa4);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, init_timestamp) == 0x424c);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, cluster_tables) == 0x42a4);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, unk_4340) == 0x4340);
+    static_assert!(offset_of!(HwDataAG15V14_8_3, gpu_keepalive_mode) == 0x4358);
+
+    // E2 StatsFrag: 0x1248 bytes, -1 fields at +0xc18/+0xc30
+    static_assert!(size_of::<GpuGlobalStatsFragG15V14_8_3>() == 0x1248);
+    static_assert!(
+        offset_of!(GpuGlobalStatsFragG15V14_8_3, stats)
+            + offset_of!(GpuStatsFragG15V14_8_3, cur_stamp_id)
+            == 0xc18
+    );
+    static_assert!(
+        offset_of!(GpuGlobalStatsFragG15V14_8_3, stats)
+            + offset_of!(GpuStatsFragG15V14_8_3, unk_id)
+            == 0xc30
+    );
+
+    static_assert!(size_of::<channels::raw::ChannelState<'static>>() == 0x30);
+}
diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs
new file mode 100644
index 0000000000000..a774f5de6573f
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/hw.rs
@@ -0,0 +1,679 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Per-SoC hardware configuration structures
+//!
+//! This module contains the definitions used to store per-GPU and per-SoC configuration data.
+
+use crate::fw::types::*;
+pub(crate) use crate::m3::identity::{GpuGen, GpuHalGeneration, GpuVariant};
+use kernel::c_str;
+use kernel::prelude::*;
+
+const MAX_POWERZONES: usize = 5;
+
+/// G15 SRAM rail floor for a device tree without `apple,min-sram-microvolt`, in microvolts: the
+/// J516S runtime ADT perf-states-sram table floors SRAM at 810 mV (7.1.12 `hw::t6030`).
+const G15_MIN_SRAM_MICROVOLT: u32 = 810_000;
+
+pub(crate) mod agx3;
+pub(crate) mod t8122;
+
+/// GPU revision enumeration. Note: Part of the UABI.
+#[derive(Debug, PartialEq, Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum GpuRevision {
+    A0 = 0x00,
+    A1 = 0x01,
+    B0 = 0x10,
+    B1 = 0x11,
+    /// The chip revision T6031 (M3 Max) reports (/arm-io chip-revision 0x12). Only the T6031
+    /// table names it; no identification path maps a register read to it yet.
+    #[allow(dead_code)]
+    B2 = 0x12,
+    C0 = 0x20,
+    C1 = 0x21,
+}
+
+/// GPU core type enumeration. Note: Part of the firmware ABI.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[repr(u32)]
+pub(crate) enum GpuCore {
+    // Unknown = 0,
+    // G5P = 1,
+    // G5G = 2,
+    // G9P = 3,
+    // G9G = 4,
+    // G10P = 5,
+    // G11P = 6,
+    // G11M = 7,
+    // G11G = 8,
+    // G12P = 9,
+    // G13P = 10,
+    G13G = 11,
+    G13S = 12,
+    G13C = 13,
+    // G14P = 14,
+    G14G = 15,
+    G14S = 16,
+    G14C = 17,
+    G14D = 18, // Split out, unlike G13D
+    // G15 firmware core type IDs (G15S = 0x17).
+    // G15M = 19,
+    // G15P_AGX2 = 20,
+    // G15P = 21,
+    G15G = 22,
+    G15S = 23,
+    /// Not confirmed on hardware yet; only the T6031 table uses it.
+    #[allow(dead_code)]
+    G15C = 24,
+}
+
+/// GPU revision ID. Note: Part of the firmware ABI.
+#[derive(Debug, PartialEq, Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum GpuRevisionID {
+    // Unknown = 0,
+    A0 = 1,
+    A1 = 2,
+    B0 = 3,
+    B1 = 4,
+    C0 = 5,
+    C1 = 6,
+}
+
+/// A single performance state of the GPU.
+#[derive(Debug)]
+pub(crate) struct PState {
+    /// Voltage in millivolts, per GPU cluster.
+    pub(crate) volt_mv: KVec<u32>,
+    /// Frequency in hertz.
+    pub(crate) freq_hz: u32,
+    /// Maximum power consumption of the GPU at this pstate, in milliwatts.
+    pub(crate) pwr_mw: u32,
+}
+
+impl PState {
+    pub(crate) fn max_volt_mv(&self) -> u32 {
+        *self.volt_mv.iter().max().expect("No voltages")
+    }
+}
+
+/// A power zone definition (we have no idea what this is but Apple puts them in the DT).
+#[allow(missing_docs)]
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct PowerZone {
+    pub(crate) target: u32,
+    pub(crate) target_offset: u32,
+    pub(crate) filter_tc: u32,
+}
+
+/// An MMIO mapping used by the firmware.
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct IOMapping {
+    /// Base physical address of the mapping.
+    pub(crate) base: usize,
+    /// Whether this mapping should be replicated to all dies
+    pub(crate) per_die: bool,
+    /// Number of mappings.
+    pub(crate) count: usize,
+    /// Size of one mapping.
+    pub(crate) size: usize,
+    /// Stride between mappings.
+    pub(crate) stride: usize,
+    /// Whether the mapping should be writable.
+    pub(crate) writable: bool,
+}
+
+impl IOMapping {
+    /// Convenience constructor for a new IOMapping.
+    pub(crate) const fn new(
+        base: usize,
+        per_die: bool,
+        count: usize,
+        size: usize,
+        stride: usize,
+        writable: bool,
+    ) -> IOMapping {
+        IOMapping {
+            base,
+            per_die,
+            count,
+            size,
+            stride,
+            writable,
+        }
+    }
+}
+
+/// Unknown HwConfigA fields that vary from SoC to SoC.
+#[allow(missing_docs)]
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct HwConfigA {
+    pub(crate) unk_87c: i32,
+    pub(crate) unk_8cc: u32,
+    pub(crate) unk_e24: u32,
+}
+
+/// Unknown HwConfigB fields that vary from SoC to SoC.
+#[allow(missing_docs)]
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct HwConfigB {
+    pub(crate) unk_454: u32,
+    pub(crate) unk_4e0: u64,
+    pub(crate) unk_534: u32,
+    pub(crate) unk_ab8: u32,
+    pub(crate) unk_abc: u32,
+    pub(crate) unk_b30: u32,
+}
+
+/// Render command configs that vary from SoC to SoC.
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct HwRenderConfig {
+    /// Vertex/tiling-related configuration register (lsb: disable clustering)
+    pub(crate) tiling_control: u32,
+}
+
+#[derive(Debug)]
+pub(crate) struct HwConfigShared2Curves {
+    pub(crate) t1_coef: u32,
+    pub(crate) t2: &'static [i16],
+    pub(crate) t3_coefs: &'static [u32],
+    pub(crate) t3_scales: &'static [u32],
+}
+
+/// Static hardware clustering configuration for multi-cluster SoCs.
+#[derive(Debug)]
+pub(crate) struct HwClusteringConfig {
+    pub(crate) meta1_blocksize: usize,
+    pub(crate) meta2_size: usize,
+    pub(crate) meta3_size: usize,
+    pub(crate) meta4_size: usize,
+    pub(crate) max_splits: usize,
+}
+
+/// Static hardware configuration for a given SoC model.
+#[derive(Debug)]
+pub(crate) struct HwConfig {
+    /// Chip ID in hex format (e.g. 0x8103 for t8103).
+    pub(crate) chip_id: u32,
+    /// GPU generation.
+    pub(crate) gpu_gen: GpuGen,
+    /// GPU variant type.
+    pub(crate) gpu_variant: GpuVariant,
+    /// GPU core type ID (as known by the firmware).
+    ///
+    /// This is a firmware-ABI value (renumbered per firmware build). It is
+    /// `None` where no firmware GpuCore id has been recovered (AGX3 parts):
+    /// consumers must fail closed rather than substitute a number.
+    pub(crate) gpu_core: Option<GpuCore>,
+
+    /// Base clock used used for timekeeping.
+    pub(crate) base_clock_hz: u32,
+    /// Output address space for the UAT on this SoC.
+    pub(crate) uat_oas: u32,
+    /// Input-address bits translated by each UAT root.
+    pub(crate) uat_ias: u8,
+    /// Number of dies on this SoC.
+    pub(crate) num_dies: u32,
+    /// Maximum number of clusters on this SoC.
+    pub(crate) max_num_clusters: u32,
+    /// Maximum number of cores per cluster for this GPU.
+    pub(crate) max_num_cores: u32,
+    /// Maximum number of frags per cluster for this GPU.
+    pub(crate) max_num_frags: u32,
+    /// Maximum number of GPs per cluster for this GPU.
+    pub(crate) max_num_gps: u32,
+
+    /// Required size of the first preemption buffer.
+    pub(crate) preempt1_size: usize,
+    /// Required size of the second preemption buffer.
+    pub(crate) preempt2_size: usize,
+    /// Required size of the third preemption buffer.
+    pub(crate) preempt3_size: usize,
+
+    /// Required size of the compute preemption buffer.
+    pub(crate) compute_preempt1_size: usize,
+
+    pub(crate) clustering: Option<HwClusteringConfig>,
+
+    /// Rendering-relevant configuration.
+    pub(crate) render: HwRenderConfig,
+
+    /// Misc HWDataA field values.
+    pub(crate) da: HwConfigA,
+    /// Misc HWDataB field values.
+    pub(crate) db: HwConfigB,
+    /// HwDataShared1.table.
+    pub(crate) shared1_tab: &'static [i32],
+    /// HwDataShared1.unk_a4.
+    pub(crate) shared1_a4: u32,
+    /// HwDataShared2.table.
+    pub(crate) shared2_tab: &'static [i32],
+    /// HwDataShared2.unk_508.
+    pub(crate) shared2_unk_508: u32,
+    /// HwDataShared2.unk_508.
+    pub(crate) shared2_curves: Option<HwConfigShared2Curves>,
+
+    /// HwDataShared3.unk_8.
+    pub(crate) shared3_unk: u32,
+    /// HwDataShared3.table.
+    pub(crate) shared3_tab: &'static [u32],
+
+    /// Globals.idle_off_standby_timer.
+    pub(crate) idle_off_standby_timer_default: u32,
+    /// Globals.unk_hws2_4.
+    pub(crate) unk_hws2_4: Option<[F32; 8]>,
+    /// Globals.unk_hws2_24.
+    pub(crate) unk_hws2_24: u32,
+    /// Globals.unk_54
+    pub(crate) global_unk_54: u16,
+
+    /// Constant related to SRAM voltages.
+    pub(crate) sram_k: F32,
+    /// Unknown per-cluster coefficients 1.
+    pub(crate) unk_coef_a: &'static [&'static [F32]],
+    /// Unknown per-cluster coefficients 2.
+    pub(crate) unk_coef_b: &'static [&'static [F32]],
+    /// Unknown table in Global struct.
+    pub(crate) global_tab: Option<&'static [u8]>,
+    /// Whether this GPU has CS/AFR performance states
+    pub(crate) has_csafr: bool,
+
+    /// Temperature sensor list (8 bits per sensor).
+    pub(crate) fast_sensor_mask: [u64; 2],
+    /// Temperature sensor list (alternate).
+    pub(crate) fast_sensor_mask_alt: [u64; 2],
+    /// Temperature sensor present bitmask.
+    pub(crate) fast_die0_sensor_present: u32,
+    /// Required MMIO mappings for this GPU/firmware.
+    pub(crate) io_mappings: &'static [Option<IOMapping>],
+    /// SRAM base
+    pub(crate) sram_base: Option<usize>,
+    /// SRAM size
+    pub(crate) sram_size: Option<usize>,
+}
+
+/// Dynamic (fetched from hardware/DT) configuration.
+#[derive(Debug)]
+pub(crate) struct DynConfig {
+    /// Base physical address of the UAT TTB (from DT reserved memory region).
+    pub(crate) uat_ttb_base: u64,
+    /// GPU ID configuration read from hardware.
+    pub(crate) id: GpuIdConfig,
+    /// Power calibration configuration for this specific chip/device.
+    pub(crate) pwr: PwrConfig,
+    /// Firmware version.
+    #[allow(dead_code)]
+    pub(crate) firmware_version: KVec<u32>,
+
+    pub(crate) hw_data_a: KVVec<u8>,
+    pub(crate) hw_data_b: KVVec<u8>,
+    pub(crate) hw_globals: KVVec<u8>,
+}
+
+/// Specific GPU ID configuration fetched from SGX MMIO registers.
+#[derive(Debug)]
+pub(crate) struct GpuIdConfig {
+    /// GPU generation (should match static config).
+    pub(crate) gpu_gen: GpuGen,
+    /// GPU variant type (should match static config).
+    pub(crate) gpu_variant: GpuVariant,
+    /// Unified shader core generation.
+    pub(crate) usc_generation: u32,
+    /// Apple GPU HAL generation (0 for the pre-numbered generations).
+    pub(crate) gpu_hal_generation: GpuHalGeneration,
+    /// GPU silicon revision.
+    pub(crate) gpu_rev: GpuRevision,
+    /// GPU silicon revision ID (firmware enum).
+    pub(crate) gpu_rev_id: GpuRevisionID,
+    /// Number of GPU dies read from the ID register.
+    pub(crate) num_dies: u32,
+    /// Total number of GPU clusters.
+    pub(crate) num_clusters: u32,
+    /// Maximum number of GPU cores per cluster.
+    pub(crate) num_cores: u32,
+    /// Number of frags per cluster.
+    pub(crate) num_frags: u32,
+    /// Number of GPs per cluster.
+    pub(crate) num_gps: u32,
+    /// Total number of active cores for the whole GPU.
+    pub(crate) total_active_cores: u32,
+    /// Mask of active cores per cluster.
+    pub(crate) core_masks: KVec<u32>,
+    /// Packed mask of all active cores.
+    pub(crate) core_masks_packed: KVec<u32>,
+
+}
+
+/// Configurable CS/AFR GPU power settings from the device tree.
+#[derive(Debug)]
+pub(crate) struct CsAfrPwrConfig {
+    /// GPU CS performance state list.
+    pub(crate) perf_states_cs: KVec<PState>,
+    /// GPU AFR performance state list.
+    pub(crate) perf_states_afr: KVec<PState>,
+
+    /// CS leakage coefficient per die.
+    pub(crate) leak_coef_cs: KVec<F32>,
+    /// AFR leakage coefficient per die.
+    pub(crate) leak_coef_afr: KVec<F32>,
+
+    /// Minimum voltage for the CS/AFR SRAM power domain in microvolts.
+    pub(crate) min_sram_microvolt: u32,
+}
+
+/// Configurable GPU power settings from the device tree.
+#[derive(Debug)]
+pub(crate) struct PwrConfig {
+    /// GPU performance state list.
+    pub(crate) perf_states: KVec<PState>,
+    /// GPU power zone list.
+    pub(crate) power_zones: KVec<PowerZone>,
+
+    /// Core leakage coefficient per cluster.
+    pub(crate) core_leak_coef: KVec<F32>,
+    /// SRAM leakage coefficient per cluster.
+    pub(crate) sram_leak_coef: KVec<F32>,
+
+    pub(crate) csafr: Option<CsAfrPwrConfig>,
+
+    /// Maximum total power of the GPU in milliwatts.
+    pub(crate) max_power_mw: u32,
+    /// Maximum frequency of the GPU in megahertz.
+    pub(crate) max_freq_mhz: u32,
+
+    /// Minimum performance state to start at.
+    pub(crate) perf_base_pstate: u32,
+    /// Maximum enabled performance state.
+    pub(crate) perf_max_pstate: u32,
+
+    /// Minimum voltage for the SRAM power domain in microvolts.
+    pub(crate) min_sram_microvolt: u32,
+
+    // Most of these fields are just named after Apple ADT property names and we don't fully
+    // understand them. They configure various power-related PID loops and filters.
+    /// Average power filter time constant in milliseconds.
+    pub(crate) avg_power_filter_tc_ms: u32,
+    /// Average power filter PID integral gain?
+    pub(crate) avg_power_ki_only: F32,
+    /// Average power filter PID proportional gain?
+    pub(crate) avg_power_kp: F32,
+    pub(crate) avg_power_min_duty_cycle: u32,
+    /// Average power target filter time constant in periods.
+    pub(crate) avg_power_target_filter_tc: u32,
+    /// "Fast die0" (temperature?) PID integral gain.
+    pub(crate) fast_die0_integral_gain: F32,
+    /// "Fast die0" (temperature?) PID proportional gain.
+    pub(crate) fast_die0_proportional_gain: F32,
+    pub(crate) fast_die0_prop_tgt_delta: u32,
+    pub(crate) fast_die0_release_temp: u32,
+    /// Delay from the fender (?) becoming idle to powerdown
+    pub(crate) fender_idle_off_delay_ms: u32,
+    /// Timeout from firmware early wake to sleep if no work was submitted (?)
+    pub(crate) fw_early_wake_timeout_ms: u32,
+    /// Delay from the GPU becoming idle to powerdown
+    pub(crate) idle_off_delay_ms: u32,
+    /// Related to the above?
+    pub(crate) idle_off_standby_timer: u32,
+    /// Percent?
+    pub(crate) perf_boost_ce_step: u32,
+    /// Minimum utilization before performance state is increased in %.
+    pub(crate) perf_boost_min_util: u32,
+    pub(crate) perf_filter_drop_threshold: u32,
+    /// Performance PID filter time constant? (periods?)
+    pub(crate) perf_filter_time_constant: u32,
+    /// Performance PID filter time constant 2? (periods?)
+    pub(crate) perf_filter_time_constant2: u32,
+    /// Performance PID integral gain.
+    pub(crate) perf_integral_gain: F32,
+    /// Performance PID integral gain 2 (?).
+    pub(crate) perf_integral_gain2: F32,
+    pub(crate) perf_integral_min_clamp: u32,
+    /// Performance PID proportional gain.
+    pub(crate) perf_proportional_gain: F32,
+    /// Performance PID proportional gain 2 (?).
+    pub(crate) perf_proportional_gain2: F32,
+    pub(crate) perf_reset_iters: u32,
+    /// Target GPU utilization for the performance controller in %.
+    pub(crate) perf_tgt_utilization: u32,
+    /// Power sampling period in milliseconds.
+    pub(crate) power_sample_period: u32,
+    /// PPM (?) filter time constant in milliseconds.
+    pub(crate) ppm_filter_time_constant_ms: u32,
+    /// PPM (?) filter PID integral gain.
+    pub(crate) ppm_ki: F32,
+    /// PPM (?) filter PID proportional gain.
+    pub(crate) ppm_kp: F32,
+    /// Power consumption filter time constant (periods?)
+    pub(crate) pwr_filter_time_constant: u32,
+    /// Power consumption filter PID integral gain.
+    pub(crate) pwr_integral_gain: F32,
+    pub(crate) pwr_integral_min_clamp: u32,
+    pub(crate) pwr_min_duty_cycle: u32,
+    pub(crate) pwr_proportional_gain: F32,
+    /// Power sample period in base clocks, used when not an integer number of ms
+    pub(crate) pwr_sample_period_aic_clks: u32,
+
+    pub(crate) se_engagement_criteria: i32,
+    pub(crate) se_filter_time_constant: u32,
+    pub(crate) se_filter_time_constant_1: u32,
+    pub(crate) se_inactive_threshold: u32,
+    pub(crate) se_ki: F32,
+    pub(crate) se_ki_1: F32,
+    pub(crate) se_kp: F32,
+    pub(crate) se_kp_1: F32,
+    pub(crate) se_reset_criteria: u32,
+    /// Shader-engine controller target (G15 runtime layout).
+    pub(crate) se_target: u32,
+}
+
+impl PwrConfig {
+    fn load_opp(
+        dev: &kernel::device::Device,
+        name: &CStr,
+        cfg: &HwConfig,
+        is_main: bool,
+    ) -> Result<KVec<PState>> {
+        let mut perf_states = KVec::new();
+
+        let node = dev.of_node().ok_or(EIO)?;
+        let opps = node.parse_phandle(name, 0).ok_or(EIO)?;
+
+        for opp in opps.children() {
+            let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?;
+            let mut volt_uv: KVec<u32> = opp.get_property(c_str!("opp-microvolt"))?;
+            let pwr_uw: u32 = if is_main {
+                opp.get_property(c_str!("opp-microwatt"))?
+            } else {
+                0
+            };
+
+            let voltage_count = if is_main {
+                cfg.max_num_clusters
+            } else {
+                cfg.num_dies
+            };
+
+            if volt_uv.len() != voltage_count as usize {
+                dev_err!(
+                    dev,
+                    "Invalid opp-microvolt length (expected {}, got {})\n",
+                    voltage_count,
+                    volt_uv.len()
+                );
+                return Err(EINVAL);
+            }
+
+            // Round up: a voltage is never published below the device tree's.
+            volt_uv.iter_mut().for_each(|a| *a = if cfg.gpu_gen == GpuGen::G15 { a.div_ceil(1000) } else { *a / 1000 });
+            let volt_mv = volt_uv;
+
+            let pwr_mw = pwr_uw / 1000;
+
+            perf_states.push(
+                PState {
+                    freq_hz: freq_hz.try_into()?,
+                    volt_mv,
+                    pwr_mw,
+                },
+                GFP_KERNEL,
+            )?;
+        }
+
+        if perf_states.is_empty() {
+            Err(EINVAL)
+        } else {
+            Ok(perf_states)
+        }
+    }
+
+    /// Load the GPU power configuration from the device tree node of `dev`.
+    pub(crate) fn load(dev: &kernel::device::Device, cfg: &HwConfig) -> Result<PwrConfig> {
+        let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?;
+        let node = dev.of_node().ok_or(EIO)?;
+
+        macro_rules! prop {
+            ($prop:expr, $default:expr) => {{
+                node.get_opt_property(c_str!($prop))
+                    .map_err(|e| {
+                        dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e);
+                        e
+                    })?
+                    .unwrap_or($default)
+            }};
+            ($prop:expr) => {{
+                node.get_property(c_str!($prop)).map_err(|e| {
+                    dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e);
+                    e
+                })?
+            }};
+        }
+
+        let pz_data = prop!("apple,power-zones", KVec::new());
+
+        if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 {
+            dev_err!(dev, "Invalid apple,power-zones value\n");
+            return Err(EINVAL);
+        }
+
+        // Kept as on the 7.1.12 kernel this runtime was qualified with: it steps by 3 over the
+        // zone count, which reads the same zones as one zone per triple for up to one zone and
+        // fewer beyond. (7.2's shared copy reads one zone per triple.)
+        let pz_count = pz_data.len() / 3;
+        let mut power_zones = KVec::new();
+        for i in (0..pz_count).step_by(3) {
+            power_zones.push(
+                PowerZone {
+                    target: pz_data[i],
+                    target_offset: pz_data[i + 1],
+                    filter_tc: pz_data[i + 2],
+                },
+                GFP_KERNEL,
+            )?;
+        }
+
+        let core_leak_coef: KVec<F32> = prop!("apple,core-leak-coef");
+        let sram_leak_coef: KVec<F32> = prop!("apple,sram-leak-coef");
+
+        if core_leak_coef.len() != cfg.max_num_clusters as usize {
+            dev_err!(dev, "Invalid apple,core-leak-coef\n");
+            return Err(EINVAL);
+        }
+        if sram_leak_coef.len() != cfg.max_num_clusters as usize {
+            dev_err!(dev, "Invalid apple,sram_leak_coef\n");
+            return Err(EINVAL);
+        }
+
+        let csafr = if cfg.has_csafr {
+            Some(CsAfrPwrConfig {
+                perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?,
+                perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?,
+                leak_coef_cs: prop!("apple,cs-leak-coef"),
+                leak_coef_afr: prop!("apple,afr-leak-coef"),
+                min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"),
+            })
+        } else {
+            None
+        };
+
+        let power_sample_period: u32 = prop!("apple,power-sample-period");
+
+        Ok(PwrConfig {
+            core_leak_coef,
+            sram_leak_coef,
+
+            max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(),
+            max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000,
+
+            perf_base_pstate: prop!("apple,perf-base-pstate", 1),
+            perf_max_pstate: perf_states.len() as u32 - 1,
+            min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 {
+                // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table.
+                // Tolerate a DT without the property on G15 only.
+                prop!("apple,min-sram-microvolt", G15_MIN_SRAM_MICROVOLT)
+            } else {
+                prop!("apple,min-sram-microvolt")
+            },
+
+            avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"),
+            avg_power_ki_only: prop!("apple,avg-power-ki-only"),
+            avg_power_kp: prop!("apple,avg-power-kp"),
+            avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"),
+            avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"),
+            fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"),
+            fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"),
+            fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0),
+            fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80),
+            fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40),
+            fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5),
+            idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2),
+            idle_off_standby_timer: prop!(
+                "apple,idleoff-standby-timer",
+                cfg.idle_off_standby_timer_default
+            ),
+            perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25),
+            perf_boost_min_util: prop!("apple,perf-boost-min-util", 100),
+            perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"),
+            perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"),
+            perf_filter_time_constant: prop!("apple,perf-filter-time-constant"),
+            perf_integral_gain2: prop!("apple,perf-integral-gain2"),
+            perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)),
+            perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"),
+            perf_proportional_gain2: prop!("apple,perf-proportional-gain2"),
+            perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)),
+            perf_reset_iters: prop!("apple,perf-reset-iters", 6),
+            perf_tgt_utilization: prop!("apple,perf-tgt-utilization"),
+            power_sample_period,
+            ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"),
+            ppm_ki: prop!("apple,ppm-ki"),
+            ppm_kp: prop!("apple,ppm-kp"),
+            pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313),
+            pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)),
+            pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0),
+            pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"),
+            pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)),
+            pwr_sample_period_aic_clks: prop!(
+                "apple,pwr-sample-period-aic-clks",
+                cfg.base_clock_hz / 1000 * power_sample_period
+            ),
+            se_engagement_criteria: prop!("apple,se-engagement-criteria", -1),
+            se_filter_time_constant: prop!("apple,se-filter-time-constant", 9),
+            se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3),
+            se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500),
+            se_ki: prop!("apple,se-ki", f32!(-50.0)),
+            se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)),
+            se_kp: prop!("apple,se-kp", f32!(-5.0)),
+            se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)),
+            se_reset_criteria: prop!("apple,se-reset-criteria", 50),
+            se_target: if cfg.gpu_gen == GpuGen::G15 { prop!("apple,se-target", 1400) } else { 1400 },
+
+            perf_states,
+            power_zones,
+            csafr,
+        })
+    }
+
+    pub(crate) fn max_frequency_khz(&self) -> u32 {
+        self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs
new file mode 100644
index 0000000000000..f5e6ad607ec25
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+
+#[cfg(not(test))]
+use crate::m3::identity::{GpuGen, GpuVariant};
+#[cfg(test)]
+#[path = "../identity.rs"]
+mod identity;
+#[cfg(test)]
+use identity::{GpuGen, GpuVariant};
+
+/// Identification-only configuration for one AGX3 SoC.
+#[derive(Debug)]
+pub(crate) struct SocConfig {
+    /// Chip ID in hex format (e.g. 0x6050 for t6050).
+    pub(crate) chip_id: u32,
+    /// Decoded design generation for this chip.
+    pub(crate) gpu_gen: GpuGen,
+    /// Decoded variant for this chip.
+    pub(crate) gpu_variant: GpuVariant,
+    /// Hardware family byte expected at `0xd04000[31:24]`.
+    pub(crate) hw_family: u8,
+    /// Hardware variant byte expected at `0xd04000[23:16]`.
+    pub(crate) hw_variant: u8,
+    /// Die count expected at `0xd04010[19:16]`.
+    pub(crate) num_dies: u32,
+    /// Input-address bits per UAT root (42 on every AGX3 part).
+    pub(crate) uat_ias: u8,
+    /// Output address space (42-bit on every AGX3 part).
+    pub(crate) uat_oas: u32,
+}
+
+const fn agx3(
+    chip_id: u32,
+    gpu_gen: GpuGen,
+    gpu_variant: GpuVariant,
+    hw_family: u8,
+    hw_variant: u8,
+    num_dies: u32,
+) -> SocConfig {
+    SocConfig {
+        chip_id,
+        gpu_gen,
+        gpu_variant,
+        hw_family,
+        hw_variant,
+        num_dies,
+        uat_ias: 42,
+        uat_oas: 42,
+    }
+}
+
+pub(crate) const T8122: SocConfig = agx3(0x8122, GpuGen::G15, GpuVariant::G, 0x7, 2, 1);
+/// Apple M3 Pro (G15S).
+pub(crate) const T6030: SocConfig = agx3(0x6030, GpuGen::G15, GpuVariant::S, 0x7, 3, 1);
+/// Apple M3 Max (G15C).
+pub(crate) const T6031: SocConfig = agx3(0x6031, GpuGen::G15, GpuVariant::C, 0x7, 4, 1);
+/// Apple M3 Ultra (G15D, two dies).
+pub(crate) const T6032: SocConfig = agx3(0x6032, GpuGen::G15, GpuVariant::D, 0x7, 4, 2);
diff --git a/drivers/gpu/drm/asahi/m3/hw/t8122.rs b/drivers/gpu/drm/asahi/m3/hw/t8122.rs
new file mode 100644
index 0000000000000..5ba6b0ef6acd6
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/hw/t8122.rs
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Hardware configuration for T8122 (M3, G15G): one die, one cluster of ten core slots.
+//!
+//! Only the M3 runtime reads this table (`m3_soc::T8122`), and it refuses a T8122 until the
+//! rest of the SoC's runtime configuration is known (`m3_soc::Soc::require_complete`). The
+//! G15 manager backend never starts on T8122.
+
+use crate::f32;
+
+use super::*;
+
+/// Fender: ADT sgx reg[1] translated through the /arm-io ranges (SGX + 0xd00000).
+const FENDER_BASE: usize = 0x2_90d0_0000;
+/// The GPU register window: ADT sgx reg[0] translated through the /arm-io ranges.
+const SGX_BASE: usize = 0x2_9000_0000;
+
+/// The firmware IO mappings, one per HwDataB slot (31 on G15). A slot is `None` when T8122 has
+/// no block for it; its record then stays zero. Every address is a CPU physical address.
+const IO_MAPPINGS: [Option<IOMapping>; 31] = [
+    // 0: Fender. The window is 0x104000 bytes: the ADT sgx reg[1] size (0x12c000) less 0x28000,
+    // the same difference as on T6030.
+    Some(IOMapping::new(FENDER_BASE, false, 1, 0x10_4000, 0, true)),
+    // 1: AIC timer, the same 16 KiB page on every SoC.
+    Some(IOMapping::new(0x2_0e10_0000, false, 1, 0x4000, 0, false)),
+    // 2: AIC software interrupts: AIC (0x2d1000000) + 0x14000, the page that holds the ADT's
+    // meta-sw-interrupt registers.
+    Some(IOMapping::new(0x2_d101_4000, false, 1, 0x4000, 0, true)),
+    // 3: RGX: the first 128 KiB of the GPU register window.
+    Some(IOMapping::new(SGX_BASE, false, 1, 0x2_0000, 0, true)),
+    None, // 4: UVD: the ADT sgx node has no third reg range
+    None, // 5: unused
+    None, // 6: display underrun workaround
+    None, // 7: analog temperature sensors
+    None, // 8: PMP doorbell
+    // 9: metrology sensors, SGX + 0xe08000.
+    Some(IOMapping::new(SGX_BASE + 0xe0_8000, false, 1, 0x8000, 0, true)),
+    // 10: GM GIFAF registers, a 4 KiB page at Fender + 0xd000.
+    Some(IOMapping::new(FENDER_BASE + 0xd000, false, 1, 0x1000, 0, true)),
+    // 11: memory cache, two instances of 0x58000 bytes (a multiple of 16 KiB), at 0x220000000
+    // and 0x222000000.
+    Some(IOMapping::new(0x2_2000_0000, false, 2, 0x5_8000, 0x200_0000, true)),
+    None, // 12: AIC banked registers
+    None, // 13: PMGR scratch
+    None, // 14: NIA idle register, die 0
+    None, // 15: NIA idle register, die 1
+    None, // 16: CRE registers
+    None, // 17: streaming codec registers
+    // 18: telemetry dashboard, written by the firmware: the 4 KiB at +0x10000 of the PMP's
+    // telemetry window (0x2d03c0000, 0x14000 bytes).
+    Some(IOMapping::new(0x2_d03d_0000, false, 1, 0x1000, 0, true)),
+    // 19: telemetry dashboard, read by the firmware: the first 8 KiB of the same window.
+    Some(IOMapping::new(0x2_d03c_0000, false, 1, 0x2000, 0, false)),
+    None, // 20: telemetry dashboard configuration
+    None, // 21
+    None, // 22
+    None, // 23: AFR registers
+    None, // 24
+    // 25: ANE doorbell, ANE + 0x45c000.
+    Some(IOMapping::new(0x3_1145_c000, false, 1, 0x4000, 0, true)),
+    // 26: PMS metrology sensors.
+    Some(IOMapping::new(0x2_d028_0000, false, 1, 0x8000, 0, false)),
+    None, // 27
+    None, // 28
+    // 29: GPU clock generator, SGX + 0xe1c000.
+    Some(IOMapping::new(SGX_BASE + 0xe1_c000, false, 1, 0x4000, 0, false)),
+    None, // 30
+];
+
+/// T8122 (M3, G15G): one die, one cluster of ten core slots, eight or ten of them active.
+pub(crate) const HWCONFIG_T8122: super::HwConfig = HwConfig {
+    chip_id: 0x8122,
+    gpu_gen: GpuGen::G15,       // ID_VERSION[31:24] == 7
+    gpu_variant: GpuVariant::G, // ID_VERSION[23:16] == 2
+    gpu_core: Some(GpuCore::G15G),
+
+    // The 24 MHz reference clock (HwDataB base_clock_khz = 24000).
+    base_clock_hz: 24_000_000,
+    // 42-bit UAT roots and output addresses, as on every AGX3 part (hw::agx3::T8122).
+    uat_ias: 42,
+    uat_oas: 42,
+    num_dies: 1,         // ID_COUNTS_1[19:16]
+    max_num_clusters: 1, // one cluster of ten core slots
+    max_num_cores: 10,
+    max_num_frags: 10, // one fragment unit per core slot
+    max_num_gps: 4,    // ID_COUNTS_2[23:16]
+
+    // Not read by the M3 runtime, which keeps its own buffer layout.
+    preempt1_size: 0,
+    preempt2_size: 0,
+    preempt3_size: 0,
+    // Compute preemption buffer: 0xd80 bytes plus 0x700 per cluster.
+    compute_preempt1_size: 0x1480,
+    clustering: None,
+    render: HwRenderConfig { tiling_control: 0 },
+
+    da: HwConfigA {
+        // Cleared by the M3 runtime's InitData in any case.
+        unk_87c: 0,
+        // Fast-die temperature target, in centidegrees Celsius (110 C, as on T6030). The J613
+        // ADT's gpu-fast-die0-target is 88; how the two relate is not known.
+        unk_8cc: 11_000,
+        unk_e24: 0,
+    },
+    db: HwConfigB {
+        // The chip revision's major number: /arm-io chip-revision 0x20 >> 4.
+        unk_454: 2,
+        // The G15 InitData builders read none of the words below.
+        unk_4e0: 0,
+        unk_534: 0,
+        unk_ab8: 0,
+        unk_abc: 0,
+        unk_b30: 1,
+    },
+    // G15 leaves the hws1/hws2/hws3 curves zero, as on T6030.
+    shared1_tab: &[],
+    shared1_a4: 0,
+    shared2_tab: &[],
+    shared2_unk_508: 0,
+    shared2_curves: None,
+    shared3_unk: 0,
+    shared3_tab: &[],
+    // Idle-off standby timer when the device tree gives none: the J613 ADT has no
+    // gpu-idleoff-standby-timer.
+    idle_off_standby_timer_default: 700,
+    // The smart idle-off curve and count, the same as T6030's.
+    unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])),
+    unk_hws2_24: 6,
+    // The M3 runtime writes 0xffff to this Globals word in any case.
+    global_unk_54: 0,
+    // SRAM power scale 1.02 for every performance state, as on T6030.
+    sram_k: f32!(1.02),
+    unk_coef_a: &[],
+    unk_coef_b: &[],
+    // No FwUtil performance tunables are written.
+    global_tab: None,
+    has_csafr: false,
+    // The fast-die sensor mask is each machine's: the M3 runtime takes it from the boot
+    // loader's apple,fast-die0-sensor-mask (the ADT's gpu-fast-die0-sensor-mask), not from here.
+    fast_sensor_mask: [0, 0],
+    fast_sensor_mask_alt: [0, 0],
+    fast_die0_sensor_present: 0,
+    io_mappings: &IO_MAPPINGS,
+    // No Fender scratch SRAM window.
+    sram_base: None,
+    sram_size: None,
+};
diff --git a/drivers/gpu/drm/asahi/m3/identity.rs b/drivers/gpu/drm/asahi/m3/identity.rs
new file mode 100644
index 0000000000000..f468272c5680e
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/identity.rs
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+
+/// Supported GPU generation enumeration. Note: Part of the UABI.
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum GpuGen {
+    G13 = 13,
+    G14 = 14,
+    G15 = 15,
+    G17 = 17,
+}
+
+/// GPU variant enumeration. Note: Part of the UABI.
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum GpuVariant {
+    P = 'P' as u32,
+    G = 'G' as u32,
+    S = 'S' as u32,
+    C = 'C' as u32,
+    D = 'D' as u32,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum GpuHalGeneration {
+    Legacy = 0,
+    Hal200 = 200,
+}
+
+/// Public architecture axes for the G15 ID-register decoder.
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub(crate) struct GpuIdentity {
+    pub(crate) gpu_gen: GpuGen,
+    pub(crate) gpu_variant: GpuVariant,
+    pub(crate) usc_generation: u32,
+    pub(crate) gpu_hal_generation: GpuHalGeneration,
+}
+
+pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option<GpuIdentity> {
+    use GpuVariant::*;
+    if family != 7 { return None; }
+    let (gpu_variant, usc_generation) = match variant {
+        0 => (P, 2),
+        2 => (G, 3),
+        3 => (S, 3),
+        4 => match num_dies { 1 => (C, 3), 2 => (D, 3), _ => return None },
+        _ => return None,
+    };
+    Some(GpuIdentity { gpu_gen: GpuGen::G15, gpu_variant, usc_generation,
+        gpu_hal_generation: GpuHalGeneration::Legacy })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn g15_variants_keep_their_public_identity() {
+        assert_eq!(decode_gpu_identity(7, 3, 1).unwrap().gpu_variant, GpuVariant::S);
+        assert_eq!(decode_gpu_identity(7, 4, 1).unwrap().gpu_variant, GpuVariant::C);
+        assert_eq!(decode_gpu_identity(7, 4, 2).unwrap().gpu_variant, GpuVariant::D);
+        assert_eq!(decode_gpu_identity(7, 2, 1).unwrap().usc_generation, 3);
+        assert!(decode_gpu_identity(7, 4, 0).is_none());
+        assert!(decode_gpu_identity(10, 2, 1).is_none());
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs
new file mode 100644
index 0000000000000..ab0e1643468f7
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/initdata.rs
@@ -0,0 +1,1135 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+#![allow(clippy::unusual_byte_groupings)]
+
+//! GPU initialization data builder.
+//!
+//! The root of all interaction between the GPU firmware and the host driver is a complex set of
+//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration
+//! and the pointers to the ring buffers and global data fields that are used for communication at
+//! runtime.
+//!
+//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values
+//! derived from observing the InitData structures that macOS generates.
+
+use crate::f32;
+use crate::m3::g15_raw::*;
+use crate::fw::types::*;
+use crate::{
+    m3::hw,
+    mmu, //
+};
+use kernel::error::{
+    Error,
+    Result, //
+};
+use kernel::prelude::*;
+use kernel::try_init;
+
+use ::pin_init;
+use ::pin_init::Init;
+
+/// Builder helper for the global GPU InitData.
+
+/// The "unknown page" of the 39-bit user layout (7.1.12 `mmu::IOVA_UNK_PAGE`): two pages below
+/// the top of the user range.
+const IOVA_UNK_PAGE: u64 = (1 << 39) - 2 * 0x4000;
+
+/// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live
+/// in Fender scratch SRAM without a backup/restore path.
+/// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the
+/// firmware treats the value as signed, and is never smaller than u32::MAX would be if it
+/// converts ms to 24 MHz ticks in 32-bit arithmetic (both wrap to ~179 s); with 64-bit
+/// arithmetic it is ~24 days.
+#[allow(dead_code)]
+pub(crate) const G15_KEEP_POWERED_MS: u32 = i32::MAX as u32;
+
+/// Which T6030 backend a G15 InitData is built for. The two differ where the manager backend
+/// keeps the values it was validated with on J516S.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub(crate) enum G15Layout {
+    /// The G15 GpuManager backend: rings in Fender SRAM, the GPU never idles off.
+    Manager,
+    /// The serialized M3 runtime backend: rings in DRAM with finite idle-off, its own GPU and
+    /// firmware VA layout, and the values its InitData carries for the power, idle and
+    /// shader-engine controllers.
+    Runtime,
+}
+
+/// G15 InitData choices that are not part of the hardware or device-tree configuration.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct G15Options {
+    /// The backend the InitData is built for.
+    pub(crate) layout: G15Layout,
+    /// Publish the performance states as two voltage-sorted tables (see [`PStateTables`]).
+    pub(crate) split_pstates: bool,
+    /// Highest performance state the firmware may use, as an index into the voltage-sorted
+    /// primary table (1 = lowest). None publishes the whole table.
+    pub(crate) cap: Option<u64>,
+    /// Firmware VA of the timestamp arena (HwDataB+0x28); None uses the manager's kernel range.
+    pub(crate) timestamp_base: Option<u64>,
+    /// Manager layout: write the PPM words and the Globals power targets as the runtime layout
+    /// does (`asahi.g15_debug` bit 55). The runtime layout always does.
+    pub(crate) reference_ppm: bool,
+    /// Runtime layout: the SoC's HwDataB configuration words and unit masks
+    /// (`m3_soc::Soc::hwdata_b`). None for the manager layout, which does not write them.
+    pub(crate) runtime_hwdata_b: Option<&'static G15RuntimeHwDataB>,
+    /// Accept operating points whose clusters have different voltages
+    /// (`m3_soc::Soc::per_cluster_voltages`); see [`PStateTables`].
+    pub(crate) per_cluster_voltages: bool,
+}
+
+/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor
+/// device-tree values: configuration words where the manager layout keeps 0 or its validated
+/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values
+/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree.
+#[derive(Debug)]
+pub(crate) struct G15RuntimeHwDataB {
+    pub(crate) unk_454: u32,
+    pub(crate) unk_464: u32,
+    pub(crate) unk_a7c: u64,
+    pub(crate) unk_a98: u32,
+    pub(crate) unk_abc: u64,
+    pub(crate) unk_ae4: u32,
+    pub(crate) unk_b20: u32,
+    pub(crate) unk_b24: u64,
+    /// +0xb40, the "UAT enabled" word of the manager layout.
+    pub(crate) unk_554: u32,
+    pub(crate) unk_17b8: u32,
+    /// Unit enable masks, +0x17c0/+0x17c8.
+    pub(crate) unit_mask_a: u64,
+    pub(crate) unit_mask_b: u32,
+    pub(crate) unk_1808: u32,
+    pub(crate) unk_1818: u32,
+}
+
+impl G15Options {
+
+    /// The published performance-state tables for `pwr`.
+    pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result<PStateTables> {
+        PStateTables::new(pwr, self.split_pstates, self.cap, self.per_cluster_voltages)
+    }
+
+    /// RuntimePointers+0x2d0, a configuration word: 4 in the runtime backend's InitData, 0 (its
+    /// validated value) for the manager.
+    pub(crate) fn runtime_pointers_2d0(&self) -> u32 {
+        match self.layout {
+            G15Layout::Manager => 0,
+            G15Layout::Runtime => 4,
+        }
+    }
+}
+
+/// Start of the USC (shader code) window in the runtime backend's GPU VA layout.
+const RUNTIME_USC_BASE: u64 = 0x10_0000_0000;
+/// The runtime backend's unknown-page address, the end of its 42-bit user VA range.
+pub(crate) const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000;
+
+/// Maximum number of entries of a firmware performance-state table, including the off state.
+const PSTATE_TABLE_ENTRIES: usize = 16;
+
+/// The performance-state tables as published to the G15 firmware.
+///
+/// The device tree lists the performance states in the bootloader's order: several states can
+/// share one voltage, and the frequencies are not sorted. The firmware takes two tables sorted
+/// by voltage: the primary one with the highest frequency of each voltage, the secondary one with
+/// the lowest, plus the device-tree index of every entry. Entry 0 is the off state in both.
+/// Every frequency and voltage comes from the device tree; nothing is interpolated.
+///
+/// With per-cluster voltages (T6031, whose clusters have their own binned voltage tables), the
+/// states are grouped by their whole voltage list instead: states that share their highest
+/// cluster voltage must have the same voltage on every cluster, the groups are ordered by that
+/// highest voltage, and every cluster's voltage must rise from one entry to the next.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct PStateTables {
+    /// Number of entries, including the off state.
+    pub(crate) len: usize,
+    /// Device-tree index of each primary-table entry.
+    pub(crate) primary: [u8; PSTATE_TABLE_ENTRIES],
+    /// Device-tree index of each secondary-table entry (split tables only).
+    pub(crate) secondary: Option<[u8; PSTATE_TABLE_ENTRIES]>,
+    /// Highest primary-table entry the firmware may select.
+    pub(crate) max: u32,
+}
+
+impl PStateTables {
+    /// The primary/secondary split of `states`: one entry per distinct voltage, ascending.
+    fn split(
+        states: &[hw::PState],
+    ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> {
+        Self::split_for(states, false)
+    }
+
+    /// [`Self::split`], accepting per-cluster voltages when `per_cluster` is set.
+    fn split_for(
+        states: &[hw::PState],
+        per_cluster: bool,
+    ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> {
+        let mut primary = [0u8; PSTATE_TABLE_ENTRIES];
+        let mut secondary = [0u8; PSTATE_TABLE_ENTRIES];
+        if states.len() < 2 || states.len() > u8::MAX as usize || states[0].freq_hz != 0 {
+            return Err(EINVAL);
+        }
+        let mut len = 1;
+        let mut last_mv = 0;
+        loop {
+            // The next voltage above `last_mv`, and its fastest and slowest states.
+            let mut next: Option<(u32, usize, usize)> = None;
+            for (i, ps) in states.iter().enumerate().skip(1) {
+                let mv = ps.max_volt_mv();
+                // Every state after the off state runs, with one voltage for all clusters
+                // unless the SoC gives each cluster its own.
+                if ps.freq_hz == 0 || (!per_cluster && ps.volt_mv.iter().any(|v| *v != mv)) {
+                    return Err(EINVAL);
+                }
+                if mv <= last_mv {
+                    continue;
+                }
+                // States of one entry run every cluster at the same voltages.
+                if let (true, Some((best, hi, _))) = (per_cluster, next) {
+                    if mv == best && states[hi].volt_mv[..] != ps.volt_mv[..] {
+                        return Err(EINVAL);
+                    }
+                }
+                next = match next {
+                    Some((best, hi, lo)) if mv == best => Some((
+                        mv,
+                        if ps.freq_hz > states[hi].freq_hz { i } else { hi },
+                        if ps.freq_hz < states[lo].freq_hz { i } else { lo },
+                    )),
+                    Some((best, _, _)) if mv > best => next,
+                    _ => Some((mv, i, i)),
+                };
+            }
+            let Some((mv, hi, lo)) = next else {
+                break;
+            };
+            if len >= PSTATE_TABLE_ENTRIES
+                || states[hi].freq_hz <= states[primary[len - 1] as usize].freq_hz
+                || states[lo].freq_hz <= states[secondary[len - 1] as usize].freq_hz
+            {
+                // Faster states must need a higher voltage in both tables.
+                return Err(EINVAL);
+            }
+            // With per-cluster voltages, every cluster's voltage rises with the entry.
+            if per_cluster && len > 1 {
+                let previous = &states[primary[len - 1] as usize].volt_mv;
+                let current = &states[hi].volt_mv;
+                if previous.len() != current.len()
+                    || previous.iter().zip(current.iter()).any(|(p, c)| c <= p)
+                {
+                    return Err(EINVAL);
+                }
+            }
+            primary[len] = hi as u8;
+            secondary[len] = lo as u8;
+            len += 1;
+            last_mv = mv;
+        }
+        // At least one running state must have a nonzero voltage; callers
+        // index the table up to len - 1 and clamp caps to 1..=len - 1.
+        if len < 2 {
+            return Err(EINVAL);
+        }
+        Ok((primary, secondary, len))
+    }
+
+    /// Build the published tables. `cap` is an index into the voltage-sorted primary table,
+    /// also when the states are published in device-tree order: the firmware may then use any
+    /// device-tree state up to the capped frequency, and the cap is translated to the highest
+    /// device-tree index whose states all stay at or below it.
+    /// `per_cluster` accepts per-cluster voltages (see [`PStateTables`]).
+    pub(crate) fn new(
+        pwr: &hw::PwrConfig,
+        split: bool,
+        cap: Option<u64>,
+        per_cluster: bool,
+    ) -> Result<Self> {
+        let states = &pwr.perf_states;
+        if split {
+            let (primary, secondary, len) = Self::split_for(states, per_cluster)?;
+            let top = len as u64 - 1;
+            let max = cap.map_or(top, |c| c.clamp(1, top)) as u32;
+            return Ok(PStateTables {
+                len,
+                primary,
+                secondary: Some(secondary),
+                max,
+            });
+        }
+        if states.len() > PSTATE_TABLE_ENTRIES {
+            return Err(EINVAL);
+        }
+        let mut primary = [0u8; PSTATE_TABLE_ENTRIES];
+        for (i, slot) in primary.iter_mut().enumerate().take(states.len()) {
+            *slot = i as u8;
+        }
+        let top = states.len() as u32 - 1;
+        let max = match cap {
+            // No voltage-sorted table can have more entries than the device tree has states,
+            // so such a cap leaves the whole table.
+            None => top,
+            Some(c) if c >= u64::from(top) => top,
+            Some(c) => {
+                let (sorted, _, sorted_len) = Self::split(states)?;
+                let limit_hz = states[sorted[c.clamp(1, sorted_len as u64 - 1) as usize] as usize].freq_hz;
+                // The longest device-tree prefix that stays at or below the capped frequency.
+                let mut max = 1;
+                while max < top && states[max as usize + 1].freq_hz <= limit_hz {
+                    max += 1;
+                }
+                max
+            }
+        };
+        Ok(PStateTables {
+            len: states.len(),
+            primary,
+            secondary: None,
+            max,
+        })
+    }
+
+    /// The device-tree state of primary-table entry `i`.
+    pub(crate) fn primary_state<'s>(&self, pwr: &'s hw::PwrConfig, i: usize) -> &'s hw::PState {
+        &pwr.perf_states[self.primary[i] as usize]
+    }
+
+    /// The frequency of the highest entry the firmware may select, in kHz.
+    pub(crate) fn max_frequency_khz(&self, pwr: &hw::PwrConfig) -> u32 {
+        self.primary_state(pwr, self.max as usize).freq_hz / 1000
+    }
+}
+
+
+/// The G15 InitData builders. The M3 runtime places the firmware objects itself, so only the
+/// builders into plain memory are kept ([`Self::g15_contents`]).
+pub(crate) struct InitDataBuilderG15V14_8_3<'a>(core::marker::PhantomData<&'a ()>);
+
+impl<'a> InitDataBuilderG15V14_8_3<'a> {
+
+    /// Create an unknown T81xx-specific data structure.
+    fn t81xx_data(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+    ) -> impl Init<raw::T81xxData> {
+        let _perf_max_pstate = dyncfg.pwr.perf_max_pstate;
+
+        pin_init::init_zeroed::<raw::T81xxData>().chain(move |_ret| {
+            match cfg.chip_id {
+                0x8103 | 0x8112 => {
+                }
+                _ => (),
+            }
+            Ok(())
+        })
+    }
+
+    /// Build the HwDataA contents (mostly power-related configuration) without allocating the
+    /// object, so that they can fill a firmware object or a plain buffer alike.
+    pub(crate) fn hwdata_a_init(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+        g15: &G15Options,
+    ) -> Result<impl Init<raw::HwDataAG15V14_8_3, Error> + 'a> {
+        let pwr = &dyncfg.pwr;
+        let period_ms = pwr.power_sample_period;
+        let period_s = F32::from(period_ms) / f32!(1000.0);
+        let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms;
+        let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms;
+        let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into();
+        let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into();
+        let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into();
+        let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into();
+        let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms;
+        let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms;
+        let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into();
+        let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into();
+
+        let base_ps = pwr.perf_base_pstate;
+        let base_ps_scaled = 100 * base_ps;
+        let tables = g15.tables(pwr)?;
+        let max_ps = tables.max;
+        let runtime = g15.layout == G15Layout::Runtime;
+        let max_ps_scaled = 100 * max_ps;
+        let boost_ps_count = if cfg.gpu_gen == hw::GpuGen::G15 {
+            max_ps.checked_sub(base_ps).ok_or(EINVAL)?
+        } else { max_ps - base_ps };
+
+        #[allow(unused_variables)]
+        let base_clock_khz = cfg.base_clock_hz / 1000;
+        let v_clocks_per_period = pwr.pwr_sample_period_aic_clks;
+
+        #[allow(unused_variables)]
+        let clocks_per_period_coarse = cfg.base_clock_hz / 1000 * pwr.power_sample_period;
+
+        Ok(
+                try_init!(raw::HwDataAG15V14_8_3 {
+                    clocks_per_period: v_clocks_per_period,
+                    clocks_per_period_2: v_clocks_per_period,
+                    pwr_status: AtomicU32::new(4),
+                    unk_10: f32!(1.0),
+                    actual_pstate: 1,
+                    tgt_pstate: 1,
+                    base_pstate_scaled: base_ps_scaled,
+                    unk_40: 1,
+                    max_pstate_scaled: max_ps_scaled,
+                    min_pstate_scaled: 100,
+                    unk_64c: 625,
+                    pwr_filter_a_neg: f32!(1.0) - pwr_filter_a,
+                    pwr_filter_a: pwr_filter_a,
+                    pwr_integral_gain: pwr.pwr_integral_gain,
+                    pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(),
+                    max_power_1: pwr.max_power_mw.into(),
+                    pwr_proportional_gain: pwr.pwr_proportional_gain,
+                    pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(),
+                    pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32,
+                    max_pstate_scaled_2: max_ps_scaled,
+                    max_power_2: pwr.max_power_mw,
+                    max_pstate_scaled_3: max_ps_scaled,
+                    ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4,
+                    ppm_filter_a_neg: f32!(1.0) - ppm_filter_a,
+                    ppm_filter_a: ppm_filter_a,
+                    ppm_ki_dt: pwr.ppm_ki * period_s,
+                    unk_6fc: f32!(65536.0),
+                    ppm_kp: pwr.ppm_kp,
+                    pwr_min_duty_cycle: pwr.pwr_min_duty_cycle,
+                    max_pstate_scaled_4: max_ps_scaled,
+                    unk_71c: f32!(0.0),
+                    max_power_3: pwr.max_power_mw,
+                    cur_power_mw_2: 0x0,
+                    ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms,
+                    ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz,
+                    perf_tgt_utilization: pwr.perf_tgt_utilization,
+                    perf_boost_min_util: pwr.perf_boost_min_util,
+                    perf_boost_ce_step: pwr.perf_boost_ce_step,
+                    perf_reset_iters: pwr.perf_reset_iters,
+                    unk_774: 6,
+                    unk_778: 1,
+                    perf_filter_drop_threshold: pwr.perf_filter_drop_threshold,
+                    perf_filter_a_neg: f32!(1.0) - perf_filter_a,
+                    perf_filter_a2_neg: f32!(1.0) - perf_filter_a2,
+                    perf_filter_a: perf_filter_a,
+                    perf_filter_a2: perf_filter_a2,
+                    perf_ki: pwr.perf_integral_gain,
+                    perf_ki2: pwr.perf_integral_gain2,
+                    perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(),
+                    unk_79c: f32!(95.0),
+                    perf_kp: pwr.perf_proportional_gain,
+                    perf_kp2: pwr.perf_proportional_gain2,
+                    boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95),
+                    base_pstate_scaled_2: base_ps_scaled,
+                    max_pstate_scaled_5: max_ps_scaled,
+                    base_pstate_scaled_3: base_ps_scaled,
+                    perf_tgt_utilization_2: pwr.perf_tgt_utilization,
+                    base_pstate_scaled_4: base_ps_scaled,
+                    unk_7fc: f32!(65536.0),
+                    pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(),
+                    max_pstate_scaled_6: max_ps_scaled.into(),
+                    max_freq_mhz: pwr.max_freq_mhz,
+                    pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle,
+                    min_pstate_scaled_4: f32!(100.0),
+                    max_pstate_scaled_7: max_ps_scaled,
+                    unk_alpha_neg: f32!(0.8),
+                    unk_alpha: f32!(0.2),
+                    fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]),
+                    fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp,
+                    unk_87c: cfg.da.unk_87c,
+                    unk_880: 0x4,
+                    unk_894: f32!(1.0),
+
+                    fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s,
+                    unk_8a8: f32!(65536.0),
+                    fast_die0_kp: pwr.fast_die0_proportional_gain,
+                    pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle,
+                    max_pstate_scaled_8: max_ps_scaled,
+                    max_pstate_scaled_9: max_ps_scaled,
+                    fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta,
+                    unk_8cc: cfg.da.unk_8cc,
+                    max_pstate_scaled_10: max_ps_scaled,
+                    max_pstate_scaled_11: max_ps_scaled,
+                    unk_c2c: 1,
+                    power_zone_count: pwr.power_zones.len() as u32,
+                    max_power_4: pwr.max_power_mw,
+                    max_power_5: pwr.max_power_mw,
+                    max_power_6: pwr.max_power_mw,
+                    avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a,
+                    avg_power_target_filter_a: avg_power_target_filter_a,
+                    avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc,
+                    avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc,
+                    avg_power_target_filter_tc_clks: period_ms
+                        * pwr.avg_power_target_filter_tc
+                        * base_clock_khz,
+                    avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods,
+                    avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a,
+                    avg_power_filter_a: avg_power_filter_a,
+                    avg_power_ki_dt: pwr.avg_power_ki_only * period_s,
+                    unk_d20: f32!(65536.0),
+                    avg_power_kp: pwr.avg_power_kp,
+                    avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle,
+                    max_pstate_scaled_12: max_ps_scaled,
+                    max_pstate_scaled_13: max_ps_scaled,
+                    max_power_7: pwr.max_power_mw.into(),
+                    max_power_8: pwr.max_power_mw,
+                    avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms,
+                    avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz,
+                    max_pstate_scaled_14: max_ps_scaled,
+                    t81xx_data <- Self::t81xx_data(cfg, dyncfg),
+                    fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]),
+                    unk_e24: cfg.da.unk_e24,
+                    unk_e28: 1,
+                    fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]),
+                    unk_163c: 1,
+                    unk_3644: 0,
+                    ..Zeroable::init_zeroed()
+                })
+                .chain(move |raw| {
+                    for i in 0..tables.len {
+                        raw.sram_k[i] = cfg.sram_k;
+                    }
+
+                    // The only constants the G15 firmware expects in the HwDataA 0x10b4 block
+                    // (+0x10f4 = 0x32, +0x11cc = {0, 65536.0}).
+                    // unk_e10_0 sits at 0x10b4 (asserted in fw/initdata.rs), so +0x40 is 0x10f4
+                    // and +0x11c is 0x11d0. The runtime layout fills the whole block below.
+                    {
+                        raw.unk_e10_0.unk_40 = 0x32;
+                        raw.unk_e10_0.unk_11c = f32!(65536.0);
+                    }
+
+                    if let Some(csafr) = pwr.csafr.as_ref() {
+                        for (i, coef) in csafr.leak_coef_afr.iter().enumerate() {
+                            raw.aux_leak_coef.cs_1[i] = *coef;
+                            raw.aux_leak_coef.cs_2[i] = *coef;
+                        }
+
+                        for (i, coef) in csafr.leak_coef_cs.iter().enumerate() {
+                            raw.aux_leak_coef.afr_1[i] = *coef;
+                            raw.aux_leak_coef.afr_2[i] = *coef;
+                        }
+                    }
+
+                    for i in 0..dyncfg.id.num_clusters as usize {
+                        if let Some(coef_a) = cfg.unk_coef_a.get(i) {
+                            (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a);
+                            (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a);
+                        }
+                        if let Some(coef_b) = cfg.unk_coef_b.get(i) {
+                            (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b);
+                            (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b);
+                        }
+                    }
+
+                    for (i, pz) in pwr.power_zones.iter().enumerate() {
+                        raw.power_zones[i].target = pz.target;
+                        raw.power_zones[i].target_off = pz.target - pz.target_offset;
+                        raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc;
+                        raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc;
+                        let filter_a = f32!(1.0) / pz.filter_tc.into();
+                        raw.power_zones[i].filter_a = filter_a;
+                        raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a;
+                        raw.power_zones[i].unk_10 = 1320000000;
+                    }
+
+                    for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() {
+                        raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 };
+                    }
+
+                    if runtime {
+                        Self::hwdata_a_runtime(raw, pwr, max_ps_scaled, clocks_per_period_coarse);
+                    }
+
+                    Ok(())
+                }))
+    }
+
+
+    /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the
+    /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device
+    /// tree), and the leakage coefficient and flag words of the G15 tail. Words that the
+    /// runtime's InitData keeps at zero are cleared.
+    fn hwdata_a_runtime(
+        raw: &mut raw::HwDataAG15V14_8_3,
+        pwr: &hw::PwrConfig,
+        max_ps_scaled: u32,
+        clocks_per_period_coarse: u32,
+    ) {
+        let period_ms = pwr.power_sample_period;
+        let period_s = F32::from(period_ms) / f32!(1000.0);
+        let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into();
+        let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into();
+        let se = &mut raw.unk_e10_0;
+        se.unk_40 = 50;
+        se.unk_44 = 1;
+        se.gpu_se_filter_a_neg = f32!(1.0) - filter_a;
+        se.gpu_se_filter_1_a_neg = f32!(1.0) - filter_1_a;
+        se.gpu_se_filter_a = filter_a;
+        se.gpu_se_filter_1_a = filter_1_a;
+        se.gpu_se_ki_dt = pwr.se_ki * period_s;
+        se.gpu_se_ki_1_dt = pwr.se_ki_1 * period_s;
+        se.unk_68 = f32!(65536.0);
+        se.gpu_se_kp = pwr.se_kp;
+        se.gpu_se_kp_1 = pwr.se_kp_1;
+        se.unk_78 = 100;
+        se.max_pstate_scaled_1 = max_ps_scaled;
+        se.min_pstate_scaled = 100;
+        se.unk_88 = f32!(8000.0);
+        // The shader-engine target (ADT gpu-se-tgt, 1400 on J514S and J516S; the device tree
+        // may override it).
+        se.se_target = pwr.se_target;
+        se.gpu_se_filter_time_constant_ms = pwr.se_filter_time_constant * period_ms;
+        se.gpu_se_filter_time_constant_1_ms = pwr.se_filter_time_constant_1 * period_ms;
+        se.gpu_se_filter_time_constant_clks =
+            U64((pwr.se_filter_time_constant * clocks_per_period_coarse).into());
+        se.gpu_se_filter_time_constant_1_clks =
+            U64((pwr.se_filter_time_constant_1 * clocks_per_period_coarse).into());
+        se.unk_b0 = f32!(65536.0);
+        se.unk_d8 = 8000;
+        se.unk_dc = 4;
+        se.gpu_se_inactive_threshold = pwr.se_inactive_threshold;
+        se.gpu_se_engagement_criteria = pwr.se_engagement_criteria;
+        se.unk_e8 = 2;
+        se.unk_ec = 4;
+        se.gpu_se_reset_criteria = pwr.se_reset_criteria;
+        se.unk_11c = f32!(65536.0);
+        se.unk_12c = 40;
+        se.max_pstate_scaled_2 = max_ps_scaled;
+
+        // No fast-die release temperature, no +0x8b8 offset and no alternate sensor mask: the
+        // T6030 device tree has no source for them.
+        raw.fast_die0_release_temp_cc = 0;
+        raw.unk_87c = 0;
+        raw.fast_die0_sensor_mask_alt = U64(0);
+        raw.unk_3640 = f32!(5.0).to_bits();
+        // The fused core leakage coefficient of this machine (device tree), first cluster table.
+        if let Some(coef) = pwr.core_leak_coef.first() {
+            raw.cluster_tables[0][0] = coef.to_bits();
+        }
+        raw.unk_4298 = 1;
+        raw.unk_429c[0] = 1;
+    }
+
+
+    // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose
+    // source is not known yet are left 0 and marked TODO.
+    // (Plain comments: the versions macro needs `fn` right after a gating #[ver].)
+    fn hwdata_b_init(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+        g15: &G15Options,
+    ) -> Result<impl Init<raw::HwDataBG15V14_8_3, Error> + 'a> {
+        let pwr = &dyncfg.pwr;
+        let tables = g15.tables(pwr)?;
+        let runtime = g15.layout == G15Layout::Runtime;
+        let runtime_hwdata_b = g15.runtime_hwdata_b;
+        let timestamp_base = match g15.timestamp_base {
+            Some(base) => base,
+            // The runtime always names its timestamp arena.
+            None => return Err(EINVAL),
+        };
+        Ok(try_init!(raw::HwDataBG15V14_8_3 {
+            // VA-layout words, +0x0/+0x8
+            unk_8: U64(0x6f_00000000),
+            unk_18: U64(0xffc00000),
+            // USC start: the G15 firmware takes 0x10_00000000 at +0x10/+0x18. Like the G13
+            // builder, the manager leaves 0: drm/asahi passes the USC base per queue.
+            unk_20: U64(0),
+            unk_28: U64(0),
+            // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its
+            // 42-bit user layout; the manager keeps a 39-bit user range and maps the page
+            // at IOVA_UNK_PAGE, so it passes that.
+            unk_30: U64(IOVA_UNK_PAGE),
+            // FW VA of the timestamp area, +0x28
+            timestamp_area_base: U64(timestamp_base),
+            // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638)
+            // are left 0, as on G13 ("TODO: yuv matrices").
+            // +0xa28..+0xa34: chip id + 3 words (G13 unk_454..45c)
+            chip_id: cfg.chip_id,
+            unk_454: cfg.db.unk_454,
+            unk_458: 0x1,
+            // +0xa38..+0xa64 (G13 unk_460..unk_48c). +0xa48/+0xa54/+0xa64 are 0.
+            // +0xa38 (GPC present, 1 on t6030) and +0xa40 (PIO supported) are 1: the
+            // firmware runs its power management, and so powers up the GPU cores for
+            // a kicked job, only if both are non-zero. +0xa50/+0xa58/+0xa60 are 1.
+            // Tested on J516S: an empty compute job completes with these values.
+            unk_460: 1,
+            unk_468: 1,
+            unk_478: 1,
+            unk_480: 1,
+            unk_488: 1,
+            // +0xa68: 0x5dc0 = 24000
+            base_clock_khz: cfg.base_clock_hz / 1000,
+            // +0xa6c: 200 on slow simulation models, otherwise 1, so 1 on t6030. The
+            // HwDataA filter constants follow +0xa70 (the sample period in ms), which
+            // stays the DT value.
+            power_sample_period: 1,
+            // +0xa70: the value HwDataA +0xea8/+0xeac multiply the filter time
+            // constants by, i.e. the sample period in ms (inferred)
+            unk_a70: pwr.power_sample_period,
+            // TODO: +0xa74, +0xa88, +0xa90, +0xaa0, +0xac4 are board/config values whose
+            // source is not known yet; left 0.
+            unk_a78: 1,
+            unk_a7c: U64(1),
+            unk_a98: 1,
+            unk_a9c: 0x1f,
+            unk_ac8: Array::new([0, 1, 1, 0]),
+            // TODO: +0xaa8 and +0xb04 are condition flags; left 0.
+            // "UAT enabled" flag, G13 unk_554 = 1. +0xb40
+            unk_554: 0x1,
+            // +0xb44..+0xb58
+            uat_ttb_base: U64(dyncfg.uat_ttb_base),
+            gpu_core_id: cfg.gpu_core.ok_or(ENODEV)? as u32,
+            gpu_rev_id: dyncfg.id.gpu_rev_id as u32,
+            num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters,
+            max_pstate: tables.len as u32 - 1,
+            // +0x17e8/+0x1800/+0x1804/+0x1810;
+            // +0x1814 = 1 as well.
+            unk_17e8: 1,
+            timer_offset: U64(0),
+            unk_1800: 1,
+            unk_1804: 1,
+            unk_1810: 1,
+            unk_1814: 1,
+            // +0x17fc = 1 on t6030.
+            unk_17fc: 1,
+            // +0x1820: 48 x 0xff
+            all_ones_masks: Array::new([0xff; 0x30]),
+            // +0x1860 = 1 & ~(debug global bit 0), so 1 without debug. (inferred)
+            unk_1860: 1,
+            ..Zeroable::init_zeroed()
+        })
+        .chain(move |raw| {
+            // Rounded up, like the operating-point voltages.
+            let min_sram_mv = pwr.min_sram_microvolt.div_ceil(1000);
+            // Core and SRAM voltage rows of one state: every column holds the state's
+            // device-tree voltage (columns beyond the device tree's repeat the first one), and
+            // the SRAM rail never goes below the device tree's floor.
+            let rows = |ps: &hw::PState| -> ([u32; 8], [u32; 8]) {
+                let mut volts = [0u32; 8];
+                for (j, v) in volts.iter_mut().enumerate() {
+                    *v = *ps.volt_mv.get(j).unwrap_or(&ps.volt_mv[0]);
+                }
+                (volts, volts.map(|mv| mv.max(min_sram_mv)))
+            };
+
+            // p-state tables at +0xb58..+0xf9b, +0xfdc, same formulas as G13. Every
+            // frequency and voltage comes from the device tree.
+            for i in 0..tables.len {
+                let ps = tables.primary_state(pwr, i);
+                let (volts, sram) = rows(ps);
+                raw.frequencies[i] = ps.freq_hz / 1000000;
+                raw.voltages[i] = volts;
+                raw.voltages_sram[i] = sram;
+                raw.sram_k[i] = cfg.sram_k;
+            }
+
+            match tables.secondary {
+                None => {
+                    // One table in device-tree order: the second frequency table repeats it,
+                    // and the index map is the identity.
+                    for i in 0..tables.len {
+                        raw.frequencies_2[i] = raw.frequencies[i];
+                    }
+                    for i in 0..16 {
+                        raw.unk_arr_0[i] = i as u32;
+                    }
+                }
+                Some(secondary) => {
+                    // The secondary table (+0xf9c), the device-tree index of every entry of
+                    // both tables (+0x10dc, +0x111c) and the third table (+0x134c, the
+                    // secondary table with one voltage column).
+                    // The base state is a device-tree index, and the HwDataA base words use it as
+                    // an index into the published table: the two must name the same state.
+                    if pwr.perf_base_pstate as usize >= tables.len
+                        || u32::from(tables.primary[pwr.perf_base_pstate as usize])
+                            != pwr.perf_base_pstate
+                    {
+                        return Err(EINVAL);
+                    }
+                    let top = tables.primary_state(pwr, tables.len - 1);
+                    let base = tables.primary_state(pwr, pwr.perf_base_pstate as usize);
+                    if top.pwr_mw == 0 || top.freq_hz <= base.freq_hz {
+                        return Err(EINVAL);
+                    }
+                    let aux = &mut raw.aux_ps_3;
+                    aux.max_pstate = tables.len as u32 - 1;
+                    for i in 0..tables.len {
+                        let ps = &pwr.perf_states[secondary[i] as usize];
+                        let (volts, sram) = rows(ps);
+                        raw.frequencies_2[i] = ps.freq_hz / 1000000;
+                        raw.unk_arr_0[i] = tables.primary[i] as u32;
+                        raw.unk_arr_0[16 + i] = secondary[i] as u32;
+                        aux.frequencies[i] = ps.freq_hz / 1000000;
+                        aux.voltages[i][0] = volts[0];
+                        aux.voltages_sram[i][0] = sram[0];
+
+                        // Maximum power and boost frequency of each primary entry, in percent
+                        // of the top entry (boost relative to the base state).
+                        let ps = tables.primary_state(pwr, i);
+                        if i > 0 {
+                            raw.rel_max_powers[i] =
+                                (100 * u64::from(ps.pwr_mw) / u64::from(top.pwr_mw)) as u32;
+                        }
+                        if ps.freq_hz > base.freq_hz {
+                            let f = u64::from(ps.freq_hz / 1000000 - base.freq_hz / 1000000);
+                            let span = u64::from(top.freq_hz / 1000000 - base.freq_hz / 1000000);
+                            raw.rel_boost_freqs[i] = (100 * f / span) as u32;
+                        }
+                    }
+                }
+            }
+
+            if let Some(csafr) = pwr.csafr.as_ref() {
+                let aux = &mut raw.aux_ps;
+                aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?;
+                aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?;
+
+                for (i, ps) in csafr.perf_states_cs.iter().enumerate() {
+                    aux.cs_frequencies[i] = ps.freq_hz / 1000000;
+                    for (j, mv) in ps.volt_mv.iter().enumerate() {
+                        let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000);
+                        aux.cs_voltages[i][j] = *mv;
+                        aux.cs_voltages_sram[i][j] = sram_mv;
+                    }
+                }
+
+                for (i, ps) in csafr.perf_states_afr.iter().enumerate() {
+                    aux.afr_frequencies[i] = ps.freq_hz / 1000000;
+                    for (j, mv) in ps.volt_mv.iter().enumerate() {
+                        let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000);
+                        aux.afr_voltages[i][j] = *mv;
+                        aux.afr_voltages_sram[i][j] = sram_mv;
+                    }
+                }
+            }
+
+            // +0x17c0 (u64) / +0x17c8 (u32): masks with the low N bits set
+            // (per-core / per-cluster enable masks, inferred).
+            // TODO: which counts these are; default total cores / clusters.
+            let low_bits = |n: u32| -> u64 {
+                if n >= 64 {
+                    u64::MAX
+                } else {
+                    (1u64 << n) - 1
+                }
+            };
+            raw.unit_mask_a = U64(low_bits(dyncfg.id.num_cores * dyncfg.id.num_clusters));
+            raw.unit_mask_b = low_bits(dyncfg.id.num_clusters.min(32)) as u32;
+
+            if runtime {
+                Self::hwdata_b_runtime(raw, runtime_hwdata_b.ok_or(EINVAL)?);
+            }
+            Ok(())
+        }))
+    }
+
+    // The runtime backend's HwDataB words: its GPU VA layout (USC base, unknown page), and the
+    // SoC's configuration words and unit masks (`words`, see `G15RuntimeHwDataB`). Those are a
+    // runtime InitData's values, not derived from the device tree: the T6030 ones hold for the
+    // boards that InitData was checked on (J514S, J516S), not necessarily for other
+    // configurations.
+    fn hwdata_b_runtime(raw: &mut raw::HwDataBG15V14_8_3, words: &G15RuntimeHwDataB) {
+        raw.unk_20 = U64(RUNTIME_USC_BASE);
+        raw.unk_28 = U64(RUNTIME_USC_BASE);
+        raw.unk_30 = U64(RUNTIME_UNKNOWN_PAGE);
+        raw.unk_454 = words.unk_454;
+        raw.unk_464 = words.unk_464;
+        raw.unk_a7c = U64(words.unk_a7c);
+        raw.unk_a98 = words.unk_a98;
+        raw.unk_abc = U64(words.unk_abc);
+        raw.unk_ae4 = words.unk_ae4;
+        raw.unk_b20 = words.unk_b20;
+        raw.unk_b24 = U64(words.unk_b24);
+        raw.unk_554 = words.unk_554;
+        raw.unk_17b8 = words.unk_17b8;
+        raw.unit_mask_a = U64(words.unit_mask_a);
+        raw.unit_mask_b = words.unit_mask_b;
+        raw.unk_1808 = words.unk_1808;
+        raw.unk_1818 = words.unk_1818;
+    }
+
+
+    // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields
+    // with a known init value are set; the rest start at 0.
+    fn globals_init(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+        g15: &G15Options,
+    ) -> Result<impl Init<raw::GlobalsG15V14_8_3, Error> + 'a> {
+        let pwr = &dyncfg.pwr;
+        let tables = g15.tables(pwr)?;
+        let runtime = g15.layout == G15Layout::Runtime;
+        let reference_ppm = g15.reference_ppm;
+        Ok(try_init!(raw::GlobalsG15V14_8_3 {
+            // TODO: 0 or 7 depending on a board condition (+0x0); 0 = timer
+            // context switch off.
+            timer_cswitch: 0,
+            // G13 unk_24 = 0, debug = 0 (inferred)
+            unk_28: 0,
+            debug: 0,
+            // +0x38: constant 0x78
+            unk_38: 0x78,
+            // G13 unk_54/56/58 moved to +0x50..+0x54 (inferred)
+            // TODO: these are board configuration values; G13 values for now.
+            unk_50: cfg.global_unk_54,
+            unk_52: 40,
+            unk_54: 0xffff,
+            // G13 unk_5e = 1 moved to +0x5a (a flag on G15) (inferred)
+            unk_5a: U32(1),
+            // +0x5e = max(HwDataB+0xa6c, 1) = 1.
+            cswitch_timer_multiplier: U32(1),
+            // Must be non-zero before work is submitted.
+            // Non-zero at boot has been accepted by the firmware on J516S.
+            command_submission_enabled: 1,
+            pending_submissions: AtomicU32::new(0),
+            // G13 values. TODO: the G15 defaults for +0x97c..+0x984 are not
+            // known.
+            progress_check_interval_3d: 40,
+            progress_check_interval_ta: 10,
+            progress_check_interval_cl: 250,
+            // G13 unk_1102c_0.._c values; same group at +0x988 (inferred)
+            unk_1102c_0: 1,
+            unk_1102c_4: 1,
+            unk_1102c_8: 100,
+            unk_1102c_c: 1,
+            // +0x9a0..+0x9a8
+            // TODO: with the rings in Fender SRAM the GPU and Fender must stay
+            // powered from firmware boot on, until an SRAM backup/restore exists.
+            // gpu.rs update_globals() keeps idle_off_delay_ms at the same value on
+            // every submit.
+            idle_off_delay_ms: AtomicU32::new(G15_KEEP_POWERED_MS),
+            fender_idle_off_delay_ms: G15_KEEP_POWERED_MS,
+            fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms,
+            // G13 values (+0x9b0/+0x9b4, set by setters)
+            cl_context_switch_timeout_ms: 40,
+            cl_kill_timeout_ms: 50,
+            // The manager leaves the FwUtil perf tunables (cfg.global_tab) and
+            // `soft_fault_settings` (+0xdf9) unwritten; the runtime layout sets them below.
+            ..Zeroable::init_zeroed()
+        })
+        .chain(move |raw| {
+            if runtime {
+                Self::globals_runtime(raw, cfg, pwr, &tables)?;
+            } else if reference_ppm {
+                Self::globals_power_targets(raw, pwr, &tables);
+            }
+            Ok(())
+        }))
+    }
+
+    // The Globals power-interface targets (maximum power from the device tree, then the highest
+    // usable performance state scaled by 100, twice) and the performance-state cap.
+    fn globals_power_targets(raw: &mut raw::GlobalsG15V14_8_3, pwr: &hw::PwrConfig, tables: &PStateTables) {
+        let max_ps_scaled = 100 * tables.max;
+        raw.power_if_target = Array::new([pwr.max_power_mw, max_ps_scaled, max_ps_scaled]);
+        raw.perf_state_cap = max_ps_scaled;
+    }
+
+    // The runtime backend's Globals: DRAM rings with finite idle-off, so the idle policy
+    // (imported by DeviceControl 0x13) and smart idle-off are on and the device-tree
+    // idle-off delays apply; the power interface targets and the performance-state cap
+    // (maximum power from the device tree, pstates scaled by 100); the smart idle-off
+    // configuration (standby timer from the device tree); and the FwUtil tunables.
+    fn globals_runtime(
+        raw: &mut raw::GlobalsG15V14_8_3,
+        cfg: &'static hw::HwConfig,
+        pwr: &hw::PwrConfig,
+        tables: &PStateTables,
+    ) -> Result {
+        raw.relaxed_cl_kill_timeout = 3000;
+        raw.debug = 1;
+        raw.smart_idle_off_enable = 1;
+        raw.unk_50 = 0xffff;
+        // The runtime enables command submission through its own queue protocol.
+        raw.command_submission_enabled = 0;
+        Self::globals_power_targets(raw, pwr, tables);
+
+        let mut idle = [0u32; 10];
+        idle[0] = pwr.idle_off_standby_timer;
+        if let Some(curve) = cfg.unk_hws2_4 {
+            for (slot, k) in idle[1..9].iter_mut().zip(curve.iter()) {
+                *slot = k.to_bits();
+            }
+        }
+        idle[9] = cfg.unk_hws2_24;
+        raw.smart_idle_off_cfg = Array::new(idle);
+        raw.ut_engagement = 1;
+        raw.clvr_engagement = 1;
+        raw.keepalive_perf_threshold_rd = 100;
+        raw.keepalive_off_threshold_rd = 100;
+
+        raw.idle_off_delay_ms = AtomicU32::new(pwr.idle_off_delay_ms);
+        raw.fender_idle_off_delay_ms = pwr.fender_idle_off_delay_ms;
+        raw.fw_early_wake_timeout_ms = pwr.fw_early_wake_timeout_ms;
+
+        // CDM backoff 4, then a u32 1 and the FwUtil perf tunables from +0x9c1.
+        raw.cdm_backoff_timeout = 4;
+        raw.unk_9bd = Array::new([1, 0, 0, 0]);
+        if let Some(tab) = cfg.global_tab {
+            let mut bytes = [0u8; 0x1c];
+            bytes.get_mut(..tab.len()).ok_or(EINVAL)?.copy_from_slice(tab);
+            raw.fwutil_default_fab_pstate = Array::new([bytes[0], bytes[1]]);
+            raw.fwutil_timer_period = bytes[2];
+            (*raw.unk_9c4)[..0x19].copy_from_slice(&bytes[3..0x1c]);
+        }
+        raw.soft_fault_settings = U32(3);
+        Ok(())
+    }
+
+
+
+    // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the
+    // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle
+    // power-off state (+0x4), which the runtime's InitData starts at 1.
+    fn status_block_fill(raw: &mut raw::StatusBlock, g15: &G15Options) {
+        raw.flags.sched_callbacks = 1;
+        if g15.layout == G15Layout::Runtime {
+            raw.noise_suppression_idle_pwroff = 1;
+        }
+    }
+
+
+    // Build the G15 P1 power-controller block without allocating it. The fields keep the order
+    // and the formulas of the G13 13.5 Globals power fields (inferred).
+    fn power_ctl_block_init(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+        g15: &G15Options,
+    ) -> impl Init<raw::PowerCtlBlock, Error> + 'a {
+        let pwr = &dyncfg.pwr;
+        let period_ms = pwr.power_sample_period;
+        let period_s = F32::from(period_ms) / f32!(1000.0);
+        let runtime = g15.layout == G15Layout::Runtime;
+        let reference_ppm = runtime || g15.reference_ppm;
+
+        try_init!(raw::PowerCtlBlock {
+            power_zone_count: pwr.power_zones.len() as u32,
+            avg_power_filter_tc_periods: pwr.avg_power_filter_tc_ms / period_ms,
+            avg_power_ki_dt: pwr.avg_power_ki_only * period_s,
+            avg_power_kp: pwr.avg_power_kp,
+            avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle,
+            avg_power_target_filter_tc: pwr.avg_power_target_filter_tc,
+            // G13 Globals unk_89bc..unk_89e0 order (inferred)
+            fast_die_temp_target: cfg.da.unk_8cc,
+            fast_die0_release_temp: 100 * pwr.fast_die0_release_temp,
+            unk_b0: cfg.da.unk_87c,
+            fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta,
+            fast_die0_kp: pwr.fast_die0_proportional_gain,
+            fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s,
+            // +0xc0/+0xc4 are board values at init (also mirrored at HwDataA
+            // +0x934/+0x93c, G13 unk_8e8/unk_8f0, which the G13 path leaves 0).
+            // TODO: real values unknown; default 0 as in the G13 HwDataA mirror.
+            unk_c0: 0,
+            unk_c4: 0,
+            // +0xc8..+0xd8, manager layout: the G13 unk_89e0 = 1 flag (on G15 a
+            // condition flag), then the PPM target and the PPM kp. The PPM controller
+            // normally receives them at runtime; the G13 Globals values keep it from
+            // starting with a zero target.
+            ppm_words: Array::new([1, pwr.max_power_mw, pwr.ppm_kp.to_bits(), 0, 0]),
+            // +0xec: 1 at init
+            se_engagement: 1,
+            ..Zeroable::init_zeroed()
+        })
+        .chain(move |raw| {
+            for (i, pz) in pwr.power_zones.iter().enumerate() {
+                raw.power_zones[i].target = pz.target;
+                raw.power_zones[i].target_off = pz.target - pz.target_offset;
+                raw.power_zones[i].filter_tc = pz.filter_tc;
+            }
+            if runtime {
+                // Runtime layout: no fast-die release temperature or +0xb0 offset (the
+                // T6030 device tree has no source for them).
+                raw.fast_die0_release_temp = 0;
+                raw.unk_b0 = 0;
+            }
+            if reference_ppm {
+                // 0 at +0xc8, then the enable flag, the PPM target and the PPM kp and ki*dt.
+                let ppm_ki_dt = pwr.ppm_ki * period_s;
+                raw.ppm_words = Array::new([
+                    0,
+                    1,
+                    pwr.max_power_mw,
+                    pwr.ppm_kp.to_bits(),
+                    ppm_ki_dt.to_bits(),
+                ]);
+            }
+            Ok(())
+        })
+    }
+
+
+
+    /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU.
+    fn uat_level_info(
+        cfg: &'static hw::HwConfig,
+        index_shift: usize,
+        num_entries: usize,
+    ) -> raw::UatLevelInfo {
+        raw::UatLevelInfo {
+            index_shift: index_shift as _,
+            unk_1: 14,
+            unk_2: 14,
+            unk_3: 8,
+            unk_4: 0x4000,
+            num_entries: num_entries as _,
+            unk_8: U64(1),
+            unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)),
+            index_mask: U64(((num_entries - 1) << index_shift) as u64),
+        }
+    }
+
+    /// The three UatLevelInfo records of InitData: L1 ("page catalogue") entries are 8 with a
+    /// 39-bit IAS and 64 with the 42-bit G15 IAS (index mask 0x3f0_0000_0000).
+    pub(crate) fn uat_levels(cfg: &'static hw::HwConfig) -> [raw::UatLevelInfo; 3] {
+        let uat_l1_entries = 1usize << (cfg.uat_ias as usize - 36);
+        [
+            Self::uat_level_info(cfg, 36, uat_l1_entries),
+            Self::uat_level_info(cfg, 25, 2048),
+            Self::uat_level_info(cfg, 14, 2048),
+        ]
+    }
+
+}
+
+/// G15 InitData contents built into plain memory, for a backend that places the firmware objects
+/// itself (the M3 runtime backend, whose object layout is fixed).
+pub(crate) struct G15Contents {
+    pub(crate) hwdata_a: KVBox<raw::HwDataAG15V14_8_3>,
+    pub(crate) hwdata_b: KVBox<raw::HwDataBG15V14_8_3>,
+    pub(crate) globals: KVBox<raw::GlobalsG15V14_8_3>,
+    pub(crate) power_ctl_block: KVBox<raw::PowerCtlBlock>,
+    pub(crate) status_block: KVBox<raw::StatusBlock>,
+    pub(crate) uat_levels: [raw::UatLevelInfo; 3],
+    /// The published performance-state tables.
+    pub(crate) tables: PStateTables,
+}
+
+impl<'a> InitDataBuilderG15V14_8_3<'a> {
+    /// Build the G15 InitData contents with the same builders the manager backend uses, into
+    /// plain memory. Pointer fields stay zero; the caller relocates them.
+    pub(crate) fn g15_contents(
+        cfg: &'static hw::HwConfig,
+        dyncfg: &'a hw::DynConfig,
+        g15: &G15Options,
+    ) -> Result<G15Contents> {
+        let mut status_block = KVBox::init(pin_init::init_zeroed(), GFP_KERNEL)?;
+        Self::status_block_fill(&mut status_block, g15);
+        Ok(G15Contents {
+            hwdata_a: KVBox::init(Self::hwdata_a_init(cfg, dyncfg, g15)?, GFP_KERNEL)?,
+            hwdata_b: KVBox::init(Self::hwdata_b_init(cfg, dyncfg, g15)?, GFP_KERNEL)?,
+            globals: KVBox::init(Self::globals_init(cfg, dyncfg, g15)?, GFP_KERNEL)?,
+            power_ctl_block: KVBox::init(Self::power_ctl_block_init(cfg, dyncfg, g15), GFP_KERNEL)?,
+            status_block,
+            uat_levels: Self::uat_levels(cfg),
+            tables: g15.tables(&dyncfg.pwr)?,
+        })
+    }
+}
+
+/// The bytes of a firmware structure built by this module.
+///
+/// The structures are plain `repr(C)` data, and every builder starts from a zeroed object, so
+/// no byte is uninitialized.
+pub(crate) fn raw_bytes<T>(value: &T) -> &[u8] {
+    // SAFETY: `value` is a live, initialized `T` (see above); the slice covers exactly it and
+    // borrows it for its lifetime.
+    unsafe { core::slice::from_raw_parts((value as *const T).cast::<u8>(), size_of::<T>()) }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs
new file mode 100644
index 0000000000000..d78b7bf2a0ddb
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs
@@ -0,0 +1,1472 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+
+use core::mem::{offset_of, size_of};
+
+use kernel::{
+    c_str, device,
+    device::Core,
+    platform,
+    prelude::*,
+    time::{delay::fsleep, Delta},
+};
+
+use crate::{
+    float::F32,
+    m3::g15_raw::raw,
+    m3::hw, m3::initdata,
+    m3::initdata::{G15Layout, G15Options},
+    m3::m3_config::Records,
+    m3::m3_firmware::Firmware,
+    m3::m3_init_layout as init,
+    m3::m3_init_storage as storage,
+    m3::m3_memory::Buffer,
+    m3::m3_params::{self, G15Debug, InitDataParam, ThermalMode},
+    m3::m3_soc::{IoMapping, PstateTable, Soc},
+    m3::t8122_hwdata,
+    m3::t8122_start::Experiment,
+};
+
+// The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage
+// coefficient fields from core_leak_coef[4] to the end of aux_leak_coef, which the G15 builder
+// leaves zero, and nothing the builder fills.
+const _: () = {
+    type A = raw::HwDataAG15V14_8_3;
+    assert!(offset_of!(A, core_leak_coef) == 0x417c);
+    assert!(offset_of!(A, sram_leak_coef) == 0x419c);
+    assert!(offset_of!(A, aux_leak_coef) == 0x41bc);
+    assert!(t8122_hwdata::OFFSET == offset_of!(A, core_leak_coef) + 4 * size_of::<F32>());
+    assert!(
+        t8122_hwdata::OFFSET + t8122_hwdata::SIZE
+            == offset_of!(A, aux_leak_coef) + size_of::<raw::AuxLeakCoef>()
+    );
+    assert!(t8122_hwdata::OFFSET + t8122_hwdata::SIZE <= size_of::<A>());
+};
+
+/// Owner indices (`m3_init_storage`) of the objects the typed builders generate.
+pub(crate) const INITDATA: usize = storage::ROOT;
+pub(crate) const RUNTIME_POINTERS: usize = storage::RUNTIME_POINTERS;
+pub(crate) const HWDATA: usize = storage::HARDWARE_DATA;
+pub(crate) const GLOBALS: usize = storage::GLOBALS;
+pub(crate) const GLOBALS_POWER: usize = storage::GLOBALS_POWER;
+pub(crate) const STATUS: usize = storage::CONTROL_REGION;
+pub(crate) const DEBUG: usize = storage::RUNTIME_FLAGS;
+/// HwDataA follows HwDataB inside the HwData object.
+pub(crate) const HWDATA_A: usize = 0x4580;
+
+/// Offset of HwDataB's IO-mapping table and the size of one entry.
+const IO_MAPPINGS: usize = offset_of!(raw::HwDataBG15V14_8_3, io_mappings);
+const IO_MAPPING_SIZE: usize = size_of::<raw::IOMapping>();
+/// Number of IO-mapping slots of HwDataB.
+const IO_MAPPING_SLOTS: usize = 31;
+
+/// The T6030 firmware IO mappings of the runtime backend (HwDataB slot, physical address, total
+/// size, element size, writable). The runtime maps IO with its own IO-mapping table
+/// (`m3_init_storage::T6030_IOMAPS`), which must cover every entry (checked when the contents
+/// are built). The physical addresses are T6030 SoC register blocks, the same ones the G15 manager
+/// maps (`hw::t6030`).
+pub(crate) const T6030_IO_MAPPINGS: [IoMapping; 15] = [
+    (0, 0x2_90d0_0000, 0x14_4000, 0x14_4000, true), // Fender
+    (1, 0x2_0e10_1000, 1, 1, false),                // AIC timer
+    (2, 0x3_5101_4000, 0x4000, 0x4000, true),       // AIC software interrupts
+    (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true),   // RGX
+    (7, 0x3_502b_c000, 0x1000, 0x1000, false),      // analog temperature sensors
+    (9, 0x2_90e0_8000, 0x8000, 0x8000, true),       // metrology sensors
+    (11, 0x2_2000_0000, 0x12_a000, 0x9_5000, true), // memory cache, two instances
+    (12, 0x3_5104_c000, 0x4000, 0x4000, false),     // AIC banked registers
+    (18, 0x3_503d_0000, 0x4000, 0x4000, true),      // telemetry dashboard
+    (19, 0x3_503c_0000, 0x4000, 0x4000, false),     // telemetry dashboard (read)
+    (20, 0x3_503d_8000, 0x4000, 0x4000, true),      // telemetry dashboard (config)
+    (23, 0x2_9300_0000, 0x40_0000, 0x40_0000, true), // AFR registers
+    (25, 0x3_0945_c000, 1, 1, true),                // ANE doorbell
+    (26, 0x3_5028_0000, 0x8000, 0x8000, false),     // PMS metrology sensors
+    (29, 0x2_90e5_c000, 0x4000, 0x4000, false),     // GFX clock generator
+];
+
+/// The T8122 firmware IO mappings of the runtime backend, in the same form: the blocks of
+/// `hw::t8122`, with the exact register, total size and element size the G15G firmware takes in
+/// each slot. The AIC timer entry is the T6030 one, the same register on every SoC.
+/// `fill_io_mappings` checks every entry against the runtime's IO maps of the SoC
+/// ([`T8122_IOMAPS`]) before writing it.
+pub(crate) const T8122_IO_MAPPINGS: [IoMapping; 12] = [
+    (0, 0x2_90d0_0000, 0x10_4000, 0x10_4000, true), // Fender
+    (1, 0x2_0e10_1000, 1, 1, false),                // AIC timer
+    (2, 0x2_d101_4048, 1, 1, true),                 // AIC software interrupt register
+    (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true),   // RGX
+    (9, 0x2_90e0_8000, 0x8000, 0x8000, true),       // metrology sensors
+    (10, 0x2_90d0_d000, 0x1000, 0x1000, true),      // GM GIFAF registers
+    (11, 0x2_2000_0000, 0xa_a000, 0x5_5000, true),  // memory cache, two instances
+    (18, 0x2_d03d_0000, 0x1000, 0x1000, true),      // telemetry dashboard
+    (19, 0x2_d03c_0000, 0x2000, 0x2000, false),     // telemetry dashboard (read)
+    (25, 0x3_1145_c000, 1, 1, true),                // ANE doorbell
+    (26, 0x2_d028_0000, 0x8000, 0x8000, false),     // PMS metrology sensors
+    (29, 0x2_90e1_c000, 0x4000, 0x4000, false),     // GPU clock generator
+];
+
+/// The T8122 runtime's IO maps (`m3_soc::T8122.iomaps`): its IO mappings, laid out as the T6030
+/// ones are.
+pub(crate) static T8122_IOMAPS: [storage::IoMap; 12] =
+    storage::pack_iomaps(&T8122_IO_MAPPINGS, storage::IOMAP_BASE);
+
+/// Whether every entry of `mappings` is a whole number of elements in the HwConfig block of the
+/// same slot, with the same writability: the check `fill_io_mappings` makes when it builds the
+/// InitData.
+const fn same_blocks(mappings: &[IoMapping], cfg: &hw::HwConfig) -> bool {
+    let mut i = 0;
+    while i < mappings.len() {
+        let (slot, phys, total, element, writable) = mappings[i];
+        if slot >= cfg.io_mappings.len() || element == 0 || total % element != 0 {
+            return false;
+        }
+        match cfg.io_mappings[slot] {
+            Some(block)
+                if block.base as u64 & !0x3fff == phys & !0x3fff && block.writable == writable => {}
+            _ => return false,
+        }
+        i += 1;
+    }
+    true
+}
+
+const _: () = assert!(same_blocks(&T8122_IO_MAPPINGS, &hw::t8122::HWCONFIG_T8122));
+
+// The T6030 IO maps are the packed IO maps of its IO mappings, the layout the T8122 start
+// experiment derives its own from (`storage::pack_iomaps`).
+const _: () = assert!(storage::same_iomaps(
+    &storage::pack_iomaps(&T6030_IO_MAPPINGS, storage::IOMAP_BASE),
+    &storage::T6030_IOMAPS
+));
+
+/// Highest performance state device-tree InitData may use without the driver's thermal limit
+/// (`asahi.m3_thermal=off`), and the runtime cap the thermal limit starts at and falls back to
+/// (the "safe cap"). The InitData's fast-die temperature controller words stay 0, so the
+/// firmware does not limit the die temperature by itself.
+pub(crate) const ADT_MAX_PSTATE_LIMIT: u32 = 5;
+
+/// The performance states the runtime backend lets the firmware use.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct PstatePolicy {
+    /// Highest entry of the published table.
+    pub(crate) table_max: u32,
+    /// Frequency of the highest entry of the published table, in MHz.
+    pub(crate) table_max_mhz: u32,
+    /// The ceiling: the highest entry the firmware is given (the InitData's highest-state
+    /// words). A reported state above it marks the GPU failed.
+    pub(crate) max: u32,
+    /// The runtime cap at boot (the Globals cap words): the highest entry the firmware may use
+    /// until the thermal limit changes it. Equal to `max` unless the thermal limit is on or
+    /// holding, which start here and never go below it.
+    pub(crate) safe: u32,
+    /// The entry requested at boot, at most `safe`.
+    pub(crate) boot: u32,
+    /// Frequency of the ceiling entry, in MHz.
+    pub(crate) max_mhz: u32,
+    /// Frequency of the `safe` entry, in MHz.
+    pub(crate) safe_mhz: u32,
+    /// Frequency of the boot entry, in MHz.
+    pub(crate) boot_mhz: u32,
+    /// Frequency of every entry of the published table, in MHz (entry 0 is the off state).
+    pub(crate) freqs: [u32; 16],
+    /// `asahi.m3_thermal`.
+    pub(crate) thermal: ThermalMode,
+}
+
+impl PstatePolicy {
+    /// The highest frequency the GPU may run at this boot, for userspace: the ceiling's when
+    /// the thermal limit is on, the runtime cap's otherwise.
+    pub(crate) fn reported_max_mhz(&self) -> u32 {
+        match self.thermal {
+            ThermalMode::On => self.max_mhz,
+            ThermalMode::Off | ThermalMode::Hold => self.safe_mhz,
+        }
+    }
+
+    /// Clamp the `asahi.m3_max_pstate` and `asahi.m3_boot_pstate` settings to the published table
+    /// of `hwdata` (the generated HwData object image), and to `ceiling` when given (the T8122
+    /// start experiment's `asahi.t8122_pstate_cap`).
+    fn new(dev: &device::Device, hwdata: &[u8], ceiling: Option<u32>) -> Result<Self> {
+        let hwb_max = offset_of!(raw::HwDataBG15V14_8_3, max_pstate);
+        let hwb_freq = offset_of!(raw::HwDataBG15V14_8_3, frequencies);
+        let table_max = read_u32(hwdata, hwb_max)?;
+        if table_max == 0 || table_max >= 16 {
+            return Err(EINVAL);
+        }
+        let freq = |i: u32| read_u32(hwdata, hwb_freq + 4 * i as usize);
+        // Unset: the thermal limit is on.
+        let thermal = m3_params::thermal_param().unwrap_or(ThermalMode::On);
+        // Unset: the whole table when the thermal limit bounds the runtime cap, the default cap
+        // otherwise.
+        let requested = m3_params::max_pstate_param().unwrap_or(match thermal {
+            ThermalMode::Hold | ThermalMode::On => u64::from(table_max),
+            ThermalMode::Off => m3_params::MAX_PSTATE_DEFAULT,
+        });
+        let mut max = requested.clamp(1, u64::from(table_max)) as u32;
+        if u64::from(max) != requested {
+            dev_info!(
+                dev,
+                "M3: asahi.m3_max_pstate={} clamped to {} (the table has states 1..={})\n",
+                requested,
+                max,
+                table_max
+            );
+        }
+        if thermal == ThermalMode::Off && max > ADT_MAX_PSTATE_LIMIT {
+            dev_info!(
+                dev,
+                "M3: performance cap {} lowered to {}: device-tree InitData leaves the fast-die temperature controller off and the thermal limit is off (asahi.m3_thermal=off)\n",
+                max,
+                ADT_MAX_PSTATE_LIMIT
+            );
+            max = ADT_MAX_PSTATE_LIMIT;
+        }
+        if let Some(ceiling) = ceiling {
+            if max > ceiling {
+                dev_info!(
+                    dev,
+                    "M3 G15G start: performance cap {} lowered to {} (asahi.t8122_pstate_cap)\n",
+                    max,
+                    ceiling
+                );
+                max = ceiling;
+            }
+        }
+        // Without the thermal limit the runtime cap is the ceiling; with it, the runtime cap
+        // starts at (and never goes below) the cap used without it.
+        let safe = match thermal {
+            ThermalMode::Off => max,
+            ThermalMode::Hold | ThermalMode::On => max.min(ADT_MAX_PSTATE_LIMIT),
+        };
+        let boot = m3_params::boot_pstate_param()
+            .unwrap_or(u64::from(safe))
+            .clamp(1, u64::from(safe)) as u32;
+        let mut freqs = [0u32; 16];
+        for (i, f) in freqs.iter_mut().enumerate().take(table_max as usize + 1) {
+            *f = freq(i as u32)?;
+        }
+        Ok(PstatePolicy {
+            table_max,
+            table_max_mhz: freqs[table_max as usize],
+            max,
+            safe,
+            boot,
+            max_mhz: freqs[max as usize],
+            safe_mhz: freqs[safe as usize],
+            boot_mhz: freqs[boot as usize],
+            freqs,
+            thermal,
+        })
+    }
+}
+
+/// The runtime backend's InitData contents: the generated object images and the
+/// performance-state policy.
+pub(crate) struct ExtendedInputs {
+    pub(crate) power: hw::PwrConfig,
+    pub(crate) sram_uv: KVec<u32>,
+    pub(crate) bases: super::m3_mmio_26::RegisterBases,
+}
+
+pub(crate) struct Contents {
+    pub(crate) extended: Option<ExtendedInputs>,
+    /// Generated images, per owner; None for owners that stay zero. Only the three image
+    /// owners' images are uploaded; the records' are checked.
+    images: KVec<Option<KVVec<u8>>>,
+    /// The performance states the firmware may use.
+    pub(crate) pstates: PstatePolicy,
+    /// The runtime's IO maps of the SoC (`Soc::iomaps`), which cover the InitData IO mappings.
+    pub(crate) iomaps: &'static [storage::IoMap],
+    /// The accepted performance-state table (`Soc::pstates`): states above the off state, and
+    /// the frequency of the highest one in MHz.
+    pub(crate) table: (u32, u32),
+    /// The HwDataB slots whose IO maps the firmware gets read-only, as a bit mask. 0 (every IO
+    /// map read-write) except in the T8122 start experiment.
+    pub(crate) read_only_slots: u32,
+    /// The HwData object's placement and mapping (`Soc::hwdata_object`).
+    pub(crate) hwdata_object: Option<crate::m3::m3_soc::HwDataObject>,
+}
+
+fn read_u32(bytes: &[u8], offset: usize) -> Result<u32> {
+    let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?;
+    Ok(u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?))
+}
+
+fn write(bytes: &mut [u8], offset: usize, data: &[u8]) -> Result {
+    bytes
+        .get_mut(offset..offset + data.len())
+        .ok_or(EINVAL)?
+        .copy_from_slice(data);
+    Ok(())
+}
+
+impl Contents {
+    /// Build the InitData contents from this board's device tree (`asahi.m3_initdata`), before
+    /// any GPU register is touched.
+    ///
+    /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122
+    /// only; it then stands in for the SoC table's missing ones.
+    pub(crate) fn select(
+        pdev: &platform::Device<Core<'_>>,
+        firmware: &Firmware,
+        soc: &Soc,
+        experiment: Option<&Experiment>,
+    ) -> Result<Self> {
+        let dev = pdev.as_ref();
+        if firmware.profile() == super::m3_profile::Profile::Extended {
+            if experiment.is_some() { return Err(EINVAL); }
+            return Self::extended(dev, soc);
+        }
+        let param = m3_params::initdata_param();
+        check_version(dev, firmware, experiment)?;
+        let iomaps = match experiment {
+            Some(e) => e.iomaps(),
+            None => soc.iomaps.ok_or(ENODEV)?,
+        };
+        storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?;
+        let images = build_images(dev, firmware, soc, experiment, iomaps)?;
+        let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?;
+        let pstates = PstatePolicy::new(dev, hwdata, experiment.map(|e| e.pstate_cap()))?;
+        let table = match soc.pstates {
+            PstateTable::Fixed { states, top_mhz } => (states, top_mhz),
+            PstateTable::DeviceTree => crate::m3::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| {
+                dev_err!(
+                    dev,
+                    "M3: the device tree's operating points give no performance-state table\n"
+                );
+                EINVAL
+            })?,
+        };
+        if (pstates.table_max, pstates.table_max_mhz) != table {
+            dev_err!(
+                dev,
+                "M3: performance-state table 1..={} up to {} MHz, the {} table is 1..={} up to {} MHz\n",
+                pstates.table_max,
+                pstates.table_max_mhz,
+                soc.name,
+                table.0,
+                table.1
+            );
+            return Err(EINVAL);
+        }
+        // The published states never go above the device tree's highest operating point.
+        if let Some(dt_khz) = crate::m3::m3_board::max_frequency_khz(pdev) {
+            if 1000 * pstates.max_mhz > dt_khz {
+                dev_err!(
+                    dev,
+                    "M3: performance cap {} MHz is above the device tree's highest operating point ({} kHz)\n",
+                    pstates.max_mhz,
+                    dt_khz
+                );
+                return Err(EINVAL);
+            }
+        }
+        dev_info!(
+            dev,
+            "M3: InitData contents: generated from the device tree (asahi.m3_initdata={}); performance states 1..={}, cap {} ({} MHz), boot request {} ({} MHz); thermal limit {}, firmware highest state {} ({} MHz)\n",
+            match param {
+                InitDataParam::Auto => "auto",
+                InitDataParam::Adt => "adt",
+            },
+            pstates.table_max,
+            pstates.safe,
+            pstates.safe_mhz,
+            pstates.boot,
+            pstates.boot_mhz,
+            match pstates.thermal {
+                ThermalMode::Off => "off",
+                ThermalMode::Hold => "holding",
+                ThermalMode::On => "on",
+            },
+            pstates.max,
+            pstates.max_mhz
+        );
+        let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0);
+        let hwdata_object = soc.hwdata_object;
+        Ok(Contents { extended: None, images, pstates, iomaps, table, read_only_slots, hwdata_object })
+    }
+
+    fn extended(dev: &device::Device, soc: &Soc) -> Result<Self> {
+        let cfg = soc.hwcfg.ok_or(ENODEV)?;
+        if soc.chip_id != 0x8122 || cfg.chip_id != 0x8122 { return Err(ENODEV); }
+        let power = hw::PwrConfig::load(dev, cfg)?;
+        let tables = initdata::PStateTables::new(&power, true, None, false)?;
+        if tables.len != 9 || tables.max != 8 || tables.max_frequency_khz(&power) != 1_338_000 {
+            return Err(EINVAL);
+        }
+        let node = dev.of_node().ok_or(ENODEV)?;
+        let opps = node.parse_phandle(c_str!("operating-points-v2"), 0).ok_or(EINVAL)?;
+        let mut sram_uv = KVec::new();
+        for opp in opps.children() {
+            let values: KVec<u32> = opp.get_property(c_str!("opp-microvolt-sram"))?;
+            if values.len() != 1 { return Err(EINVAL); }
+            sram_uv.push(values[0], GFP_KERNEL)?;
+        }
+        if sram_uv.len() != power.perf_states.len() { return Err(EINVAL); }
+        let block = |index: usize| -> Result<u64> {
+            Ok(cfg.io_mappings.get(index).and_then(|value| value.as_ref()).ok_or(EINVAL)?.base as u64)
+        };
+        let bases = super::m3_mmio_26::RegisterBases {
+            gpu: soc.windows.sgx, interrupt: block(2)?,
+            memory: block(11)?.checked_add(0x1c4000).ok_or(EOVERFLOW)?,
+            telemetry_write: block(18)?, telemetry_read: block(19)?,
+            neural: block(25)?, metrology: block(26)?,
+        };
+        let mut freqs = [0; 16];
+        for (index, value) in freqs.iter_mut().enumerate().take(tables.len) {
+            *value = tables.primary_state(&power, index).freq_hz / 1_000_000;
+        }
+        // This profile enables the native firmware's thermal/PID controllers.
+        // The legacy host-only thermal governor must not replace those inputs.
+        let pstates = PstatePolicy { table_max: 8, table_max_mhz: 1338,
+            max: 8, safe: 8, boot: 1, max_mhz: 1338, safe_mhz: 1338,
+            boot_mhz: freqs[1], freqs, thermal: ThermalMode::Off };
+        let mut images = KVec::new();
+        for _ in 0..super::m3_init_26::COUNT { images.push(None, GFP_KERNEL)?; }
+        Ok(Self { extended: Some(ExtendedInputs { power, sram_uv, bases }),
+            images, pstates, iomaps: &[], table: (8, 1338), read_only_slots: 0, hwdata_object: None })
+    }
+
+    /// Check the contents before they are uploaded.
+    pub(crate) fn check_for_upload(&self) -> Result {
+        let count = if self.extended.is_some() { super::m3_init_26::COUNT } else { storage::COUNT };
+        if self.images.len() == count {
+            Ok(())
+        } else {
+            Err(EINVAL)
+        }
+    }
+
+    /// The initial contents of owner `index`, allocated as `a`: its generated image, or nothing
+    /// (zeroed) for owners without one.
+    pub(crate) fn initial<'a>(&'a self, index: usize, a: &storage::Allocation) -> Result<&'a [u8]> {
+        if a.initial == storage::Initial::Zero {
+            return Ok(&[]);
+        }
+        let image = self.images.get(index).and_then(|i| i.as_deref()).ok_or(EINVAL)?;
+        if image.len() != a.size {
+            return Err(EINVAL);
+        }
+        Ok(image)
+    }
+}
+
+/// The fixed allocation of owner `index`, as a firmware region moved by `delta`.
+fn allocation_region(index: usize, delta: u64) -> Result<init::Region> {
+    let a = storage::allocation(index).map_err(|_| EINVAL)?;
+    init::Region::new(a.address.checked_add(delta).ok_or(EINVAL)?, a.size).map_err(|_| EINVAL)
+}
+
+/// The records the runtime constructs, encoded for its fixed allocation table moved by `delta`.
+pub(crate) fn allocation_records(delta: u64) -> Result<Records> {
+    Records::encode(|index| allocation_region(index, delta))
+}
+
+/// The version word of the constructed InitData root.
+pub(crate) fn constructed_version() -> Result<u64> {
+    let records = allocation_records(0)?;
+    Ok(u64::from_le_bytes(records.root[..8].try_into().map_err(|_| EINVAL)?))
+}
+
+/// Refuse to go on when the InitData root the runtime constructs carries another version than
+/// the loaded firmware expects. The T8122 start experiment gives the firmware the version of
+/// `asahi.t8122_initdata_version` instead (`m3_config::Config::new` writes it into the root).
+fn check_version(
+    dev: &device::Device,
+    firmware: &Firmware,
+    experiment: Option<&Experiment>,
+) -> Result {
+    let version = constructed_version()?;
+    if version != firmware.initdata_magic && experiment.is_some() {
+        dev_warn!(
+            dev,
+            "M3 G15G start: InitData version {:#x} (asahi.t8122_initdata_version) replaces the constructed {:#x}\n",
+            firmware.initdata_magic,
+            version
+        );
+        return Ok(());
+    }
+    if version != firmware.initdata_magic {
+        dev_err!(
+            dev,
+            "M3: the constructed InitData version {:#x} is not the one the loaded firmware expects ({:#x})\n",
+            version,
+            firmware.initdata_magic
+        );
+        return Err(ENODEV);
+    }
+    Ok(())
+}
+
+/// Whether the runtime probe stops before the GPU coprocessor starts (`asahi.g15_debug` bit
+/// 53). Logs the stop.
+pub(crate) fn stop_before_asc(dev: &device::Device) -> bool {
+    let stop = m3_params::g15_debug(G15Debug::StopBeforeAsc);
+    if stop {
+        dev_info!(
+            dev,
+            "M3: asahi.g15_debug bit 53: stopping before the GPU coprocessor starts; the firmware was not started\n"
+        );
+    }
+    stop
+}
+
+/// The GPU identity the runtime backend admits for `soc` (its SGX ID words, checked in
+/// `m3_device` before the firmware starts): on T6030, G15, variant S, revision B1, one die with
+/// two clusters of ten core slots.
+fn soc_identity(
+    soc: &Soc,
+    cfg: &'static hw::HwConfig,
+    rev_id: Option<hw::GpuRevisionID>,
+) -> hw::GpuIdConfig {
+    hw::GpuIdConfig {
+        gpu_gen: hw::GpuGen::G15,
+        gpu_variant: soc.gpu_variant,
+        usc_generation: 3,
+        gpu_hal_generation: hw::GpuHalGeneration::Legacy,
+        gpu_rev: soc.gpu_revision,
+        gpu_rev_id: rev_id.unwrap_or(soc.gpu_revision_id),
+        num_dies: cfg.num_dies,
+        num_clusters: cfg.max_num_clusters,
+        num_cores: cfg.max_num_cores,
+        num_frags: cfg.max_num_frags,
+        num_gps: cfg.max_num_gps,
+        total_active_cores: 0,
+        core_masks: KVec::new(),
+        core_masks_packed: KVec::new(),
+    }
+}
+
+pub(crate) const BOARD_DATA: [usize; 3] = [HWDATA, GLOBALS, GLOBALS_POWER];
+
+/// The pointers of the constructed records that the typed firmware structures place: record
+/// owner, offset, target owner, offset into the target.
+const RECORD_POINTERS: [(usize, usize, usize, usize); 7] = {
+    type InitData = raw::InitDataG15V14_8_3<'static>;
+    type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>;
+    [
+        (INITDATA, offset_of!(InitData, runtime_pointers), RUNTIME_POINTERS, 0),
+        (INITDATA, offset_of!(InitData, globals), GLOBALS, 0),
+        (INITDATA, offset_of!(InitData, debug_block), DEBUG, 0),
+        (INITDATA, offset_of!(InitData, status_block), STATUS, 0),
+        (INITDATA, offset_of!(InitData, power_ctl_block), GLOBALS_POWER, 0),
+        (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_b), HWDATA, 0),
+        (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_a), HWDATA, HWDATA_A),
+    ]
+};
+
+/// The bytes of a constructed record, by owner.
+pub(crate) fn record_bytes(records: &Records, owner: usize) -> Option<&[u8]> {
+    match owner {
+        INITDATA => Some(&records.root[..]),
+        RUNTIME_POINTERS => Some(&records.runtime[..]),
+        STATUS => Some(&records.control[..]),
+        _ => None,
+    }
+}
+
+fn check_layout() -> Result {
+    type InitData = raw::InitDataG15V14_8_3<'static>;
+    type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>;
+    let records = allocation_records(0)?;
+    let pointer = |owner: usize, offset: usize| -> Option<u64> {
+        let bytes = record_bytes(&records, owner)?.get(offset..offset + 8)?;
+        Some(u64::from_le_bytes(bytes.try_into().ok()?))
+    };
+    let mut ok = true;
+    for (owner, offset, target, at) in RECORD_POINTERS {
+        let a = storage::allocation(target).map_err(|_| EINVAL)?;
+        ok &= at < a.size && pointer(owner, offset) == Some(a.address + at as u64);
+    }
+    for index in 0..storage::COUNT {
+        let a = storage::allocation(index).map_err(|_| EINVAL)?;
+        let board = BOARD_DATA.contains(&index);
+        ok &= board == (a.initial != storage::Initial::Zero);
+    }
+    let size = |index: usize| storage::allocation(index).map_or(0, |a| a.size);
+    let ok = ok
+        && size(INITDATA) >= size_of::<InitData>()
+        && size(RUNTIME_POINTERS) >= offset_of!(RuntimePointers, hwdata_a) + 8
+        && HWDATA_A >= size_of::<raw::HwDataBG15V14_8_3>()
+        && size(HWDATA) >= HWDATA_A + size_of::<raw::HwDataAG15V14_8_3>()
+        && size(GLOBALS) >= size_of::<raw::GlobalsG15V14_8_3>()
+        && size(GLOBALS_POWER) >= size_of::<raw::PowerCtlBlock>()
+        && size(STATUS) >= size_of::<raw::StatusBlock>()
+        && size(DEBUG) >= size_of::<raw::DebugBlock>();
+    if ok { Ok(()) } else { Err(EINVAL) }
+}
+
+/// Fill HwDataB's IO-mapping table (the runtime fills the virtual addresses when it maps them),
+/// checking every entry against the runtime's IO mapping of the same slot (`iomaps`).
+fn fill_io_mappings(
+    dev: &device::Device,
+    cfg: &'static hw::HwConfig,
+    experiment: Option<&Experiment>,
+    mappings: &[IoMapping],
+    iomaps: &[storage::IoMap],
+    hwdata: &mut [u8],
+) -> Result {
+    for (i, &(slot, phys, total, element, writable)) in mappings.iter().enumerate() {
+        if slot >= 31 || mappings[..i].iter().any(|&(other, ..)| other == slot) {
+            return Err(EINVAL);
+        }
+        let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE;
+        let virt = entry + offset_of!(raw::IOMapping, virt_addr);
+        // The same register block as the manager's table.
+        let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| {
+            let base = match experiment {
+                Some(e) => e.io_block(slot, m.base as u64),
+                None => m.base as u64,
+            };
+            base & !0x3fff == phys & !0x3fff && m.writable == writable
+        });
+        let covered = iomaps.iter().any(|io| {
+            io.slot == slot
+                && io.pointer_field() == Ok(virt)
+                && io.covers(phys, total)
+        });
+        if !same_block || !covered || element == 0 || total % element != 0 {
+            dev_err!(dev, "M3: IO mapping slot {} ({:#x}) does not match the object layout\n", slot, phys);
+            return Err(EINVAL);
+        }
+        write(hwdata, entry + offset_of!(raw::IOMapping, phys_addr), &phys.to_le_bytes())?;
+        write(hwdata, entry + offset_of!(raw::IOMapping, total_size), &total.to_le_bytes())?;
+        write(hwdata, entry + offset_of!(raw::IOMapping, element_size), &element.to_le_bytes())?;
+        write(
+            hwdata,
+            entry + offset_of!(raw::IOMapping, readwrite),
+            &u64::from(writable).to_le_bytes(),
+        )?;
+    }
+    // Every IO mapping the runtime makes must belong to one of the entries above.
+    for io in iomaps.iter() {
+        if !mappings.iter().any(|&(slot, ..)| slot == io.slot) {
+            dev_err!(dev, "M3: IO mapping slot {} ({:#x}) has no device-tree InitData entry\n", io.slot, io.physical);
+            return Err(EINVAL);
+        }
+    }
+    Ok(())
+}
+
+pub(crate) fn check_upload(
+    dev: &device::Device,
+    firmware: &Firmware,
+    iomaps: &[storage::IoMap],
+    objects: &mut [Buffer],
+) -> Result {
+    let mut bad = 0u32;
+    let version = objects.get_mut(INITDATA).ok_or(EINVAL)?.read_u64(0)?;
+    if version != firmware.initdata_magic {
+        dev_err!(
+            dev,
+            "M3: InitData self-check: version {:#x}, the loaded firmware expects {:#x}\n",
+            version,
+            firmware.initdata_magic
+        );
+        bad += 1;
+    }
+    for (owner, offset, target, at) in RECORD_POINTERS {
+        let expected = objects.get(target).ok_or(EINVAL)?.va() + at as u64;
+        let value = objects.get_mut(owner).ok_or(EINVAL)?.read_u64(offset)?;
+        if value != expected {
+            dev_err!(
+                dev,
+                "M3: InitData self-check: owner {} +{:#x} holds {:#x}, not owner {} +{:#x} ({:#x})\n",
+                owner,
+                offset,
+                value,
+                target,
+                at,
+                expected
+            );
+            bad += 1;
+        }
+    }
+    let hwdata = objects.get_mut(HWDATA).ok_or(EINVAL)?;
+    for slot in 0..IO_MAPPING_SLOTS {
+        let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE;
+        let phys = hwdata.read_u64(entry + offset_of!(raw::IOMapping, phys_addr))?;
+        let virt = hwdata.read_u64(entry + offset_of!(raw::IOMapping, virt_addr))?;
+        let total = u64::from(hwdata.read_u32(entry + offset_of!(raw::IOMapping, total_size))?);
+        let ok = match iomaps.iter().find(|io| io.slot == slot) {
+            Some(io) => {
+                phys == io.physical + io.offset as u64
+                    && virt == io.address + io.offset as u64
+                    && total != 0
+                    && phys + total <= io.physical + io.size as u64
+            }
+            None => phys == 0 && virt == 0,
+        };
+        if !ok {
+            dev_err!(
+                dev,
+                "M3: InitData self-check: IO mapping slot {} describes {:#x}+{:#x} at {:#x}, which the runtime does not map there\n",
+                slot,
+                phys,
+                total,
+                virt
+            );
+            bad += 1;
+        }
+    }
+    let region = hwdata.read_u64(storage::GPU_REGION_PHYSICAL)?;
+    if region != firmware.resources.regions[0].base {
+        dev_err!(
+            dev,
+            "M3: InitData self-check: GPU region {:#x}, the resources admitted {:#x}\n",
+            region,
+            firmware.resources.regions[0].base
+        );
+        bad += 1;
+    }
+    if bad != 0 {
+        dev_err!(dev, "M3: InitData self-check failed ({} findings); not publishing the InitData\n", bad);
+        return Err(EINVAL);
+    }
+    dev_info!(
+        dev,
+        "M3: InitData self-check passed: version {:#x}, {} record pointers, {} IO mappings, GPU region {:#x}\n",
+        version,
+        RECORD_POINTERS.len(),
+        iomaps.len(),
+        region
+    );
+    Ok(())
+}
+
+/// The whole part of a positive `F32`, for logs.
+fn whole(v: F32) -> u32 {
+    let bits = v.to_bits();
+    match ((bits >> 23) & 0xff) as i32 - 127 {
+        e if e < 0 => 0,
+        e if e > 31 => u32::MAX,
+        e if e > 23 => ((bits & 0x7f_ffff) | 0x80_0000) << (e - 23),
+        e => ((bits & 0x7f_ffff) | 0x80_0000) >> (23 - e),
+    }
+}
+
+/// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported
+/// them. Returns them when the boot loader's switch is set. With the switch set and no two
+/// valid values, refuses: the boot loader then used neither the fuse nor the stand-in.
+fn leak_fuse(dev: &device::Device, lf: &crate::m3::m3_soc::LeakFuse) -> Result<Option<[F32; 2]>> {
+    let chosen = kernel::of::chosen();
+    let switch = chosen
+        .as_ref()
+        .and_then(|c| c.get_property::<KVec<u8>>(lf.switch).ok())
+        .is_some_and(|v| v.as_slice() == b"1\0");
+    let values: Option<KVec<F32>> = chosen.as_ref().and_then(|c| c.get_property(lf.values).ok());
+    // Positive, finite, nonzero.
+    let valid = |v: &F32| {
+        let b = v.to_bits();
+        b != 0 && b >> 31 == 0 && (b >> 23) & 0xff != 0xff
+    };
+    let pair = values
+        .as_ref()
+        .filter(|v| v.len() == 2 && v.iter().all(valid))
+        .map(|v| [v[0], v[1]]);
+    let names = (lf.values.to_str().unwrap_or("?"), lf.switch.to_str().unwrap_or("?"));
+    match (pair, switch) {
+        (Some([a, b]), true) => {
+            dev_info!(
+                dev,
+                "M3: GPU leakage from the boot loader's fuse values: {} and {} (/chosen/{}, {} set)\n",
+                whole(a),
+                whole(b),
+                names.0,
+                names.1
+            );
+            Ok(Some([a, b]))
+        }
+        (Some([a, b]), false) => {
+            dev_info!(
+                dev,
+                "M3: the boot loader's GPU leakage fuse values are {} and {} (/chosen/{}); not used: {} is not set\n",
+                whole(a),
+                whole(b),
+                names.0,
+                names.1
+            );
+            Ok(None)
+        }
+        (None, true) => {
+            dev_err!(
+                dev,
+                "M3: {} is set, but /chosen/{} does not hold two valid values; GPU startup disabled\n",
+                names.1,
+                names.0
+            );
+            Err(EINVAL)
+        }
+        (None, false) => {
+            if values.is_some() {
+                dev_warn!(dev, "M3: /chosen/{} is malformed; ignored\n", names.0);
+            }
+            Ok(None)
+        }
+    }
+}
+
+/// A zeroed image of owner `index`.
+fn zeroed(index: usize) -> Result<KVVec<u8>> {
+    let size = storage::allocation(index).map_err(|_| EINVAL)?.size;
+    Ok(KVVec::from_elem(0u8, size, GFP_KERNEL)?)
+}
+
+/// Build the images of the generated owners from the device tree.
+fn build_images(
+    dev: &device::Device,
+    firmware: &Firmware,
+    soc: &Soc,
+    experiment: Option<&Experiment>,
+    iomaps: &[storage::IoMap],
+) -> Result<KVec<Option<KVVec<u8>>>> {
+    check_layout().inspect_err(|_| {
+        dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n")
+    })?;
+    let cfg: &'static hw::HwConfig = soc.hwcfg.ok_or(ENODEV)?;
+    let io_mappings = match experiment {
+        Some(e) => e.io_mappings(),
+        None => soc.io_mappings.ok_or(ENODEV)?,
+    };
+    let runtime_hwdata_b = match experiment {
+        Some(e) => e.hwdata_b(),
+        None => soc.hwdata_b.ok_or(ENODEV)?,
+    };
+    let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| {
+        dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e)
+    })?;
+    // The boot loader's decoded leakage fuse, when its switch makes it this boot's leakage: value
+    // 1 is the core leakage coefficient (the boot loader then writes it as apple,core-leak-coef
+    // too), value 2 goes to HwDataA's third leakage table below.
+    let fuse = match soc.leak_fuse {
+        Some(lf) => leak_fuse(dev, &lf)?,
+        None => None,
+    };
+    if let Some([core, _]) = fuse {
+        match pwr.core_leak_coef.first_mut() {
+            Some(c) if c.to_bits() != core.to_bits() => {
+                dev_info!(
+                    dev,
+                    "M3: core leakage {} replaces the device tree's {}\n",
+                    whole(core),
+                    whole(*c)
+                );
+                *c = core;
+            }
+            Some(_) => {}
+            None => return Err(EINVAL),
+        }
+    }
+    // A SoC whose power model is still a stand-in caps the power target: every operating point's
+    // power is scaled by the same factor, so the relative powers stay the boot loader's.
+    let power_cap_mw = soc.power_target_cap_mw;
+    if let Some(cap) = power_cap_mw {
+        let max = pwr.max_power_mw;
+        if max > cap {
+            for ps in pwr.perf_states.iter_mut() {
+                ps.pwr_mw = (u64::from(ps.pwr_mw) * u64::from(cap) / u64::from(max)) as u32;
+            }
+            pwr.max_power_mw = pwr.perf_states.iter().map(|ps| ps.pwr_mw).max().unwrap_or(cap);
+            dev_info!(
+                dev,
+                "M3: GPU power target capped at {} mW (the device tree's operating points reach {} mW)\n",
+                pwr.max_power_mw,
+                max
+            );
+        }
+    }
+    let node = dev.of_node().ok_or(ENODEV)?;
+    let dyncfg = hw::DynConfig {
+        uat_ttb_base: firmware.resources.regions[0].base,
+        id: soc_identity(soc, cfg, None),
+        pwr,
+        firmware_version: node
+            .get_property::<KVec<u32>>(c_str!("apple,firmware-version"))
+            .unwrap_or_default(),
+        hw_data_a: KVVec::new(),
+        hw_data_b: KVVec::new(),
+        hw_globals: KVVec::new(),
+    };
+    let g15 = G15Options {
+        layout: G15Layout::Runtime,
+        split_pstates: true,
+        // The cap is applied to the uploaded objects, for either source (`publish_pstates`).
+        cap: None,
+        timestamp_base: Some(crate::m3::agx_memory::TIMESTAMP_RANGE.start),
+        reference_ppm: true,
+        runtime_hwdata_b: Some(runtime_hwdata_b),
+        per_cluster_voltages: soc.per_cluster_voltages,
+    };
+    let c = initdata::InitDataBuilderG15V14_8_3::g15_contents(cfg, &dyncfg, &g15).inspect_err(|e| {
+        dev_err!(dev, "M3: cannot build InitData from the device tree ({:?})\n", e)
+    })?;
+
+    let mut images = KVec::new();
+    for _ in 0..storage::COUNT {
+        images.push(None, GFP_KERNEL)?;
+    }
+
+    // InitData: the magic of the identified firmware, the host-mapped allocations flag and the
+    // UAT description; the pointers are the constructed record's.
+    type InitData = raw::InitDataG15V14_8_3<'static>;
+    let mut root = zeroed(INITDATA)?;
+    write(&mut root, offset_of!(InitData, ver_info), &firmware.initdata_magic.to_le_bytes())?;
+    write(&mut root, offset_of!(InitData, unk_2c), &1u32.to_le_bytes())?;
+    write(&mut root, offset_of!(InitData, uat_page_size), &0x4000u16.to_le_bytes())?;
+    write(&mut root, offset_of!(InitData, uat_page_bits), &[14, 3])?;
+    for (i, level) in c.uat_levels.iter().enumerate() {
+        let at = offset_of!(InitData, uat_level_info) + i * size_of::<raw::UatLevelInfo>();
+        write(&mut root, at, initdata::raw_bytes(level))?;
+    }
+    images[INITDATA] = Some(root);
+
+    // RuntimePointers: only its configuration words; everything else is a pointer.
+    type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>;
+    let mut rt = zeroed(RUNTIME_POINTERS)?;
+    write(&mut rt, offset_of!(RuntimePointers, unk_2d0), &g15.runtime_pointers_2d0().to_le_bytes())?;
+    write(&mut rt, offset_of!(RuntimePointers, cswitch_hist_ctl), &[0xff])?;
+    images[RUNTIME_POINTERS] = Some(rt);
+
+    // HwData: HwDataB, then HwDataA.
+    let mut hwdata = zeroed(HWDATA)?;
+    write(&mut hwdata, 0, initdata::raw_bytes(&*c.hwdata_b))?;
+    // The T8122 start experiment's unit masks (`asahi.t8122_unit_mask_a`/`_b`).
+    if let Some(e) = experiment {
+        type B = raw::HwDataBG15V14_8_3;
+        let (a, b) = e.unit_masks();
+        write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?;
+        write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?;
+        dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b);
+    }
+    write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?;
+    // The system counter value at InitData creation, the base of the firmware's first power and
+    // energy interval.
+    let now: u64;
+    // SAFETY: reading the architectural counter has no side effects and is allowed at EL1/EL2.
+    unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) };
+    write(
+        &mut hwdata,
+        HWDATA_A + offset_of!(raw::HwDataAG15V14_8_3, init_timestamp),
+        &now.to_le_bytes(),
+    )?;
+    // The SoC's MTR sensor masks, where its table gives them.
+    if let Some(m) = soc.mtr_masks {
+        type A = raw::HwDataAG15V14_8_3;
+        for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] {
+            write(&mut hwdata, HWDATA_A + at, &m.fast_die.to_le_bytes())?;
+        }
+        write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &m.alarm.to_le_bytes())?;
+        dev_info!(
+            dev,
+            "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n",
+            m.fast_die,
+            m.alarm
+        );
+    }
+    // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first
+    // table holds the core coefficient, from the builder).
+    if let Some([_, second]) = fuse {
+        type A = raw::HwDataAG15V14_8_3;
+        let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>();
+        write(&mut hwdata, at, &second.to_bits().to_le_bytes())?;
+    }
+    // The SoC's own values for words the shared builder writes otherwise.
+    let (hwdata_words, globals_words) = (soc.hwdata_words, soc.globals_words);
+    for &(at, value) in hwdata_words {
+        write(&mut hwdata, at, &value.to_le_bytes())?;
+    }
+    // T8122's exact chip/firmware identity selects this block on Aurora; keep the
+    // existing constant model instead of restoring omacom's diagnostic opt-out.
+    if crate::m3::t8122_start::is_t8122(soc)
+        && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT)
+        && cfg.gpu_gen == hw::GpuGen::G15
+    {
+        let a = hwdata
+            .get_mut(HWDATA_A..HWDATA_A + size_of::<raw::HwDataAG15V14_8_3>())
+            .ok_or(EINVAL)?;
+        t8122_hwdata::write(a).map_err(|()| EINVAL)?;
+        dev_info!(dev, "M3 G15G: wrote identity-selected HwDataA +0x418c block\n");
+    }
+    // On T8122, log and require the two HwDataB words the firmware's power management depends
+    // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero.
+    if core::ptr::eq(soc, &crate::m3::m3_soc::T8122) {
+        type B = raw::HwDataBG15V14_8_3;
+        let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?;
+        let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?;
+        dev_info!(dev, "M3: HwDataB +0xa38 {:#x}, +0xa40 {:#x}\n", a38, a40);
+        if a38 == 0 || a40 == 0 {
+            dev_err!(
+                dev,
+                "M3: HwDataB +0xa38 or +0xa40 is 0; the firmware would not power the GPU cores\n"
+            );
+            return Err(EINVAL);
+        }
+    }
+    fill_io_mappings(dev, cfg, experiment, io_mappings, iomaps, &mut hwdata)?;
+    images[HWDATA] = Some(hwdata);
+
+    let mut globals = zeroed(GLOBALS)?;
+    write(&mut globals, 0, initdata::raw_bytes(&*c.globals))?;
+    // The gate of the firmware's frequency-feedback cap, per SoC.
+    let (ut, given) = m3_params::ut_engagement(soc);
+    write(&mut globals, offset_of!(raw::GlobalsG15V14_8_3, ut_engagement), &ut.to_le_bytes())?;
+    for &(at, value) in globals_words {
+        write(&mut globals, at, &value.to_le_bytes())?;
+    }
+    if !hwdata_words.is_empty() || !globals_words.is_empty() {
+        dev_info!(
+            dev,
+            "M3: {} HwData and {} Globals words set from the {} table\n",
+            hwdata_words.len(),
+            globals_words.len(),
+            soc.name
+        );
+    }
+    if ut != 1 || given {
+        dev_info!(
+            dev,
+            "M3: Globals ut_engagement={}{}\n",
+            ut,
+            if given { " (asahi.m3_ut_engagement)" } else { "" }
+        );
+    }
+    if crate::m3::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals)
+        .map_err(|_|EINVAL)? {
+        dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n");
+    }
+    images[GLOBALS] = Some(globals);
+
+    let mut power = zeroed(GLOBALS_POWER)?;
+    write(&mut power, 0, initdata::raw_bytes(&*c.power_ctl_block))?;
+    images[GLOBALS_POWER] = Some(power);
+
+    let mut status = zeroed(STATUS)?;
+    write(&mut status, 0, initdata::raw_bytes(&*c.status_block))?;
+    images[STATUS] = Some(status);
+
+    let top = c.tables.primary_state(&dyncfg.pwr, c.tables.len - 1);
+    dev_info!(
+        dev,
+        "M3: device-tree InitData: {} performance states in two voltage-sorted tables from {} device-tree states, top {} MHz at {} mV, SRAM floor {} mV, maximum power {} mW\n",
+        c.tables.len - 1,
+        dyncfg.pwr.perf_states.len() - 1,
+        top.freq_hz / 1_000_000,
+        top.max_volt_mv(),
+        dyncfg.pwr.min_sram_microvolt.div_ceil(1000),
+        dyncfg.pwr.max_power_mw
+    );
+    log_tables(dev, &c.hwdata_b, c.tables.len);
+    Ok(images)
+}
+
+/// Log the published performance-state tables, one line per entry: the primary table (fastest
+/// state per voltage) and the secondary one (slowest), each with the device-tree index of the
+/// entry, and the third table.
+fn log_tables(dev: &device::Device, b: &raw::HwDataBG15V14_8_3, len: usize) {
+    let uniform = |row: &[u32; 8]| row.iter().all(|v| *v == row[0]);
+    for i in 1..len.min(16) {
+        let aux = &b.aux_ps_3;
+        let ok = uniform(&b.voltages[i]) && uniform(&b.voltages_sram[i]);
+        dev_info!(
+            dev,
+            "M3: performance state {}: primary dt {} {} MHz core {} mV SRAM {} mV; secondary dt {} {} MHz; third {} MHz core {} mV SRAM {} mV{}\n",
+            i,
+            b.unk_arr_0[i],
+            b.frequencies[i],
+            b.voltages[i][0],
+            b.voltages_sram[i][0],
+            b.unk_arr_0[16 + i],
+            b.frequencies_2[i],
+            aux.frequencies[i],
+            aux.voltages[i][0],
+            aux.voltages_sram[i][0],
+            if ok { "" } else { " (columns differ)" }
+        );
+    }
+}
+
+/// HwDataA words that hold the highest usable performance state scaled by 100.
+const HWA_MAX_SCALED: [usize; 15] = {
+    type A = raw::HwDataAG15V14_8_3;
+    let se = offset_of!(A, unk_e10_0);
+    [
+        offset_of!(A, max_pstate_scaled),
+        offset_of!(A, max_pstate_scaled_2),
+        offset_of!(A, max_pstate_scaled_3),
+        offset_of!(A, max_pstate_scaled_4),
+        offset_of!(A, max_pstate_scaled_5),
+        offset_of!(A, max_pstate_scaled_7),
+        offset_of!(A, max_pstate_scaled_8),
+        offset_of!(A, max_pstate_scaled_9),
+        offset_of!(A, max_pstate_scaled_10),
+        offset_of!(A, max_pstate_scaled_11),
+        offset_of!(A, max_pstate_scaled_12),
+        offset_of!(A, max_pstate_scaled_13),
+        offset_of!(A, max_pstate_scaled_14),
+        se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_1),
+        se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_2),
+    ]
+};
+
+/// HwDataA words that hold the base (lowest settled) performance state scaled by 100.
+const HWA_BASE_SCALED: [usize; 4] = {
+    type A = raw::HwDataAG15V14_8_3;
+    [
+        offset_of!(A, base_pstate_scaled),
+        offset_of!(A, base_pstate_scaled_2),
+        offset_of!(A, base_pstate_scaled_3),
+        offset_of!(A, base_pstate_scaled_4),
+    ]
+};
+
+/// Offsets in Globals of the runtime cap words: the second and third power-interface targets and
+/// the performance-state cap, each the highest usable state scaled by 100.
+const GLOBALS_RUNTIME_CAP: [usize; 3] = {
+    type G = raw::GlobalsG15V14_8_3;
+    let power_if_target = offset_of!(G, power_if_target);
+    [power_if_target + 4, power_if_target + 8, offset_of!(G, perf_state_cap)]
+};
+
+pub(crate) fn publish_pstates(
+    dev: &device::Device,
+    hwdata: &mut Buffer,
+    globals: &mut Buffer,
+    p: &PstatePolicy,
+) -> Result {
+    use crate::float::F32;
+    type A = raw::HwDataAG15V14_8_3;
+    let a = |offset: usize| HWDATA_A + offset;
+    let full = 100 * p.table_max;
+    let base_scaled = hwdata.read_u32(a(offset_of!(A, base_pstate_scaled)))?;
+    let globals_max = GLOBALS_RUNTIME_CAP;
+
+    let mut expected = true;
+    for off in HWA_MAX_SCALED {
+        expected &= hwdata.read_u32(a(off))? == full;
+    }
+    for off in HWA_BASE_SCALED {
+        expected &= hwdata.read_u32(a(off))? == base_scaled;
+    }
+    for off in globals_max {
+        expected &= globals.read_u32(off)? == full;
+    }
+    expected &= hwdata.read_u32(a(offset_of!(A, max_pstate_scaled_6)))? == F32::from(full).to_bits();
+    let base = base_scaled / 100;
+    if !expected
+        || base_scaled % 100 != 0
+        || base == 0
+        || base > p.safe
+        || p.boot > p.safe
+        || p.safe > p.max
+    {
+        dev_err!(dev, "M3: unexpected performance-state words in the InitData; not starting the firmware\n");
+        return Err(EINVAL);
+    }
+
+    let ceiling = 100 * p.max;
+    let max_power = F32::from_bits(hwdata.read_u32(a(offset_of!(A, max_power_1)))?);
+    let min_duty = hwdata.read_u32(a(offset_of!(A, pwr_min_duty_cycle)))?;
+    for off in HWA_MAX_SCALED {
+        hwdata.u32(a(off), ceiling)?;
+    }
+    hwdata.u32(a(offset_of!(A, max_pstate_scaled_6)), F32::from(ceiling).to_bits())?;
+    hwdata.u32(a(offset_of!(A, pwr_pstate_related_k)), (-F32::from(ceiling) / max_power).to_bits())?;
+    hwdata.u32(
+        a(offset_of!(A, pwr_pstate_max_dc_offset)),
+        (min_duty as i32 - ceiling as i32) as u32,
+    )?;
+    hwdata.u32(
+        a(offset_of!(A, boost_state_unk_k)),
+        (F32::from(p.max - base) / crate::f32!(0.95)).to_bits(),
+    )?;
+    for off in globals_max {
+        globals.u32(off, 100 * p.safe)?;
+    }
+
+    for off in [offset_of!(A, actual_pstate), offset_of!(A, tgt_pstate)] {
+        hwdata.u32(a(off), p.boot)?;
+    }
+    let settle = base;
+    dev_info!(
+        dev,
+        "M3: firmware performance request: state {} ({} MHz), base state {}, cap state {} ({} MHz) of 1..={}\n",
+        p.boot,
+        p.boot_mhz,
+        settle,
+        p.safe,
+        p.safe_mhz,
+        p.table_max
+    );
+    if p.safe != p.max {
+        dev_info!(
+            dev,
+            "M3: firmware highest state {} ({} MHz); the runtime cap starts at {} ({} MHz)\n",
+            p.max,
+            p.max_mhz,
+            p.safe,
+            p.safe_mhz
+        );
+    }
+    Ok(())
+}
+
+/// Set the runtime cap: the Globals runtime cap words, which the firmware keeps reading while it
+/// runs, so the store alone moves the cap, with no message. `cap` must be a state of the
+/// published table.
+pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result {
+    if cap == 0 || cap >= 16 {
+        return Err(EINVAL);
+    }
+    for off in GLOBALS_RUNTIME_CAP {
+        globals.u32(off, 100 * cap)?;
+    }
+    crate::m3::agx_memory::publish();
+    Ok(())
+}
+
+/// The integer part of a non-negative, finite F32 given by its bits; None otherwise.
+fn f32_bits_to_u32(bits: u32) -> Option<u32> {
+    let exp = (bits >> 23) & 0xff;
+    if bits >> 31 != 0 || exp == 0xff {
+        return None;
+    }
+    if exp < 127 {
+        return Some(0);
+    }
+    let mant = u64::from((bits & 0x7f_ffff) | 0x80_0000);
+    let shift = exp - 127;
+    let value = if shift >= 23 {
+        mant.checked_shl(shift - 23)?
+    } else {
+        mant >> (23 - shift)
+    };
+    u32::try_from(value).ok()
+}
+
+/// One reading of the performance state: SGX+0xe01000, and the HwDataA actual and current
+/// state and frequency (MHz, F32 bits) the firmware publishes.
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct PstateSample {
+    register: u32,
+    actual: u32,
+    current: u32,
+    freq_bits: u32,
+}
+
+/// Number of lines logging changes of the performance state per boot.
+const PSTATE_LOG_LINES: u32 = 64;
+
+/// Number of lines warning about a reported state above the runtime cap per boot.
+const RUNTIME_CAP_WARN_LINES: u32 = 16;
+
+/// Checks the performance state the firmware reports against the ceiling and the runtime cap,
+/// after boot and after every job.
+///
+/// The firmware publishes its actual and current state and its frequency in HwDataA. A
+/// frequency above the ceiling entry's, or, when no running frequency is reported, a state in
+/// the published table above the ceiling, that is still there when read again 1 ms later fails
+/// the check. The same above the runtime cap (when it is below the ceiling) is reported to the
+/// caller, which decides whether it persists too long (the thermal limit). Values outside the
+/// published table are logged and not enforced: they are not states of this table.
+/// SGX+0xe01000 is logged next to them but not enforced, as the index space of that register is
+/// not established on every board. The published tables list device-tree operating points only,
+/// so the check guards the cap, not the voltages.
+pub(crate) struct PstateWatch {
+    policy: PstatePolicy,
+    last: Option<PstateSample>,
+    lines: u32,
+    warn_lines: u32,
+    register_noted: bool,
+    unknown_noted: bool,
+    excused_noted: bool,
+}
+
+impl PstateWatch {
+    pub(crate) fn new(policy: PstatePolicy) -> Self {
+        PstateWatch {
+            policy,
+            last: None,
+            lines: 0,
+            warn_lines: 0,
+            register_noted: false,
+            unknown_noted: false,
+            excused_noted: false,
+        }
+    }
+
+    fn sample(hwdata: &mut Buffer, register: u32) -> Result<PstateSample> {
+        type A = raw::HwDataAG15V14_8_3;
+        Ok(PstateSample {
+            register,
+            actual: hwdata.read_u32(HWDATA_A + offset_of!(A, actual_pstate))?,
+            current: hwdata.read_u32(HWDATA_A + offset_of!(A, cur_pstate))?,
+            freq_bits: hwdata.read_u32(HWDATA_A + offset_of!(A, freq_mhz))?,
+        })
+    }
+
+    /// Frequency of entry `state` of the published table, in MHz.
+    fn mhz(&self, state: u32) -> u32 {
+        self.policy.freqs.get(state as usize).copied().unwrap_or(0)
+    }
+
+    /// The reported frequency in MHz, if it is a running frequency of the published table.
+    fn freq_mhz(&self, s: &PstateSample) -> Option<u32> {
+        f32_bits_to_u32(s.freq_bits).filter(|mhz| *mhz > 0 && *mhz <= self.policy.table_max_mhz)
+    }
+
+    /// Whether a reported state lies within the published table but above `cap`.
+    fn state_above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> {
+        let above = |v: u32| v > cap && v <= self.policy.table_max;
+        if above(s.actual) {
+            Some(("actual state", s.actual))
+        } else if above(s.current) {
+            Some(("current state", s.current))
+        } else {
+            None
+        }
+    }
+
+    /// The reported value above `cap` that lies within the published table, if any. The
+    /// frequency is the physical quantity: a state above the cap is excused when the reported
+    /// frequency is a running frequency within the cap (the state words then use another
+    /// index space), and counts otherwise.
+    fn above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> {
+        match self.freq_mhz(s) {
+            Some(mhz) if mhz > self.mhz(cap) => Some(("frequency (MHz)", mhz)),
+            Some(_) => None,
+            None => self.state_above(s, cap),
+        }
+    }
+
+    /// Whether a reported value is not a state or frequency of the published table.
+    fn unknown(&self, s: &PstateSample) -> bool {
+        let p = &self.policy;
+        s.actual > p.table_max
+            || s.current > p.table_max
+            || f32_bits_to_u32(s.freq_bits).is_none_or(|mhz| mhz > p.table_max_mhz)
+    }
+
+    fn log(&self, dev: &device::Device, what: &str, s: &PstateSample, cap: u32) {
+        dev_info!(
+            dev,
+            "M3: performance state {}: firmware actual {} current {} frequency {} MHz, sgx+0xe01000 {:#x}; cap state {} ({} MHz)\n",
+            what,
+            s.actual,
+            s.current,
+            f32_bits_to_u32(s.freq_bits).unwrap_or(u32::MAX),
+            s.register,
+            cap,
+            self.mhz(cap)
+        );
+    }
+
+    /// Read the existing firmware state words and SGX state register without
+    /// requesting a state, publishing a cap, or enforcing a sample.
+    pub(crate) fn log_active(&self, dev: &device::Device, hwdata: &mut Buffer,
+                            register: u32, cap: u32, sequence: u32) -> Result {
+        let s = Self::sample(hwdata, register)?;
+        dev_info!(dev,
+            "M3_ACTIVE_POWER sample={} actual={} current={} freq_bits={:#x} sgx={:#x} cap={} cap_mhz={}\n",
+            sequence, s.actual, s.current, s.freq_bits, s.register, cap, self.mhz(cap));
+        Ok(())
+    }
+
+    /// Check the reported performance state. `register` reads SGX+0xe01000; `what` names the
+    /// point of the check for the log; `cap` is the runtime cap the firmware may use now
+    /// (clamped to the ceiling). Returns ERANGE when the firmware stays above the ceiling;
+    /// otherwise whether it stays above `cap`, and the highest state it reports (0 if neither
+    /// state word is a state of the published table).
+    pub(crate) fn check(
+        &mut self,
+        dev: &device::Device,
+        hwdata: &mut Buffer,
+        register: impl Fn() -> Result<u32>,
+        what: &str,
+        cap: u32,
+    ) -> Result<(bool, u32)> {
+        let ceiling = self.policy.max;
+        let cap = cap.clamp(1, ceiling);
+        let mut s = Self::sample(hwdata, register()?)?;
+        let mut over = false;
+        if let Some((field, value)) = self.above(&s, cap) {
+            if cap == ceiling || self.warn_lines < RUNTIME_CAP_WARN_LINES {
+                self.warn_lines = self.warn_lines.saturating_add(u32::from(cap != ceiling));
+                dev_warn!(
+                    dev,
+                    "M3: firmware reports {} {} above the cap {} ({} MHz) {}; reading again\n",
+                    field,
+                    value,
+                    cap,
+                    self.mhz(cap),
+                    what
+                );
+            }
+            fsleep(Delta::from_millis(1));
+            s = Self::sample(hwdata, register()?)?;
+            if let Some((field, value)) = self.above(&s, ceiling) {
+                self.log(dev, what, &s, cap);
+                dev_err!(
+                    dev,
+                    "M3: firmware {} {} stays above the cap {} ({} MHz); marking the GPU failed\n",
+                    field,
+                    value,
+                    ceiling,
+                    self.mhz(ceiling)
+                );
+                return Err(ERANGE);
+            }
+            over = self.above(&s, cap).is_some();
+        }
+        if !self.excused_noted {
+            if let Some((field, value)) = self.state_above(&s, cap) {
+                if !over {
+                    self.excused_noted = true;
+                    dev_info!(
+                        dev,
+                        "M3: firmware {} {} is above the cap {} but its frequency is within it {}; not enforced\n",
+                        field,
+                        value,
+                        cap,
+                        what
+                    );
+                }
+            }
+        }
+        if !self.unknown_noted && self.unknown(&s) {
+            self.unknown_noted = true;
+            dev_info!(
+                dev,
+                "M3: firmware performance words outside the published table (states 1..={}, up to {} MHz) {}; not enforced\n",
+                self.policy.table_max,
+                self.policy.table_max_mhz,
+                what
+            );
+            self.log(dev, what, &s, cap);
+        }
+        if !self.register_noted && s.register & 0xf > cap {
+            self.register_noted = true;
+            dev_info!(
+                dev,
+                "M3: sgx+0xe01000 reads {:#x}, above the cap as a table index; logged only\n",
+                s.register
+            );
+        }
+        let key = PstateSample {
+            register: s.register & 0xf,
+            ..s
+        };
+        if self.last != Some(key) {
+            if self.lines < PSTATE_LOG_LINES {
+                self.log(dev, what, &s, cap);
+            } else if self.lines == PSTATE_LOG_LINES {
+                dev_info!(dev, "M3: further performance state changes are not logged\n");
+            }
+            self.lines = self.lines.saturating_add(1);
+            self.last = Some(key);
+        }
+        let peak = [s.actual, s.current]
+            .into_iter()
+            .filter(|v| *v <= self.policy.table_max)
+            .max()
+            .unwrap_or(0);
+        Ok((over, peak))
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs
index 851b4e79f421e..27295660640ae 100644
--- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs
@@ -83,10 +83,52 @@ pub(crate) const fn packet_retired(pending: bool, active: bool) -> bool {
     !pending && !active
 }
 
+/// A failed packet that no batch ever included can drop its VM job guard: the firmware never saw
+/// it. Any other keeps the guard, and its VM stays quarantined, until the runtime drops it.
+pub(crate) const fn release_unpublished(may_be_published: bool) -> bool {
+    !may_be_published
+}
+
 pub(crate) const fn retain_runtime(stop_failed: bool, gpu_pending: bool) -> bool {
     stop_failed || gpu_pending
 }
 
+/// One firmware start per boot. `stop_asc` only clears RUN: it doesn't reset the firmware or
+/// give back a reusable RTKit or InitData state, and TTBAT context 0 still names the lower root
+/// the first start adopted, which is freed once ASC has stopped. Claimed just before the first
+/// ASC start and never released.
+pub(crate) struct StartLatch(core::sync::atomic::AtomicBool);
+impl StartLatch {
+    pub(crate) const fn new() -> Self { Self(core::sync::atomic::AtomicBool::new(false)) }
+    /// True for the first claim only.
+    pub(crate) fn claim(&self) -> bool { !self.0.swap(true, core::sync::atomic::Ordering::AcqRel) }
+}
+
+/// Why a start was refused before ASC.
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum StartRefusal {
+    /// The module is being removed: nothing is claimed or started.
+    Removing,
+    /// This boot's start already happened: the extra module reference is dropped again.
+    AlreadyStarted,
+}
+
+/// Pins the module, then claims `latch`. The pin comes first, so a probe racing an ordinary
+/// rmmod fails closed instead of reviving the module's reference count. A start that loses the
+/// claim drops only its own pin; the winner keeps its pin for good, so a reload can't forget the
+/// latch.
+pub(crate) fn claim_start(latch: &StartLatch, pin: impl FnOnce() -> bool, unpin: impl FnOnce())
+    -> Result<(), StartRefusal> {
+    if !pin() {
+        return Err(StartRefusal::Removing);
+    }
+    if !latch.claim() {
+        unpin();
+        return Err(StartRefusal::AlreadyStarted);
+    }
+    Ok(())
+}
+
 #[derive(Default)]
 pub(crate) struct PassCosts { recent: [Option<(u64, u64)>; 8] }
 impl PassCosts {
diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs
new file mode 100644
index 0000000000000..5d4ea4277b618
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_board.rs
@@ -0,0 +1,558 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Board facts for the M3 runtime, read from the hardware and the device tree instead of being
+//! fixed to one board. The per-SoC facts are in [`crate::m3::m3_soc`].
+
+use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi};
+
+use crate::m3::m3_resources::{
+    reserved_resource,
+    Region,
+    Resources, //
+};
+use crate::m3::m3_soc::Soc;
+
+/// Whether the root node's compatible list contains `compatible`.
+pub(crate) fn board_is(compatible: &[u8]) -> bool {
+    let Some(root) = kernel::of::root() else {
+        return false;
+    };
+    let Ok(board) = root.get_property::<KVec<u8>>(c_str!("compatible")) else {
+        return false;
+    };
+    board.split(|b| *b == 0).any(|s| s == compatible)
+}
+
+/// Whether this is a board the M3 runtime is validated on (`soc.validated_boards`).
+/// `asahi.m3_expose=auto` registers the render node and `asahi.m3_backend=auto` starts the
+/// runtime only on these.
+pub(crate) fn runtime_validated_board(soc: &Soc) -> bool {
+    soc.validated_boards.iter().any(|board| board_is(board))
+}
+
+/// The number of core slots of `soc` (2 x 10 on T6030, 4 x 10 on T6031).
+pub(crate) fn core_slots(soc: &Soc) -> u32 {
+    soc.cores_per_cluster * soc.clusters
+}
+
+/// Whether the core-enable mask describes a usable GPU of `soc`: at least one core, and no core
+/// outside its core slots. The mask is the fused core-enable words from SGX+0xe01500, the second
+/// word as bits 32..63 when the SoC has more than 32 core slots. It varies with the SKU (0x6f5fc
+/// on a 14-core T6030, 0x7fdff on an 18-core one); absent cores must never be enabled.
+pub(crate) fn core_mask_valid(soc: &Soc, mask: u64) -> bool {
+    let slots = core_slots(soc);
+    mask != 0 && slots <= 64 && (slots == 64 || mask >> slots == 0)
+}
+
+/// Per-cluster UAPI core masks for a core-enable mask of `soc`: consecutive fields of
+/// `cores_per_cluster` bits, cluster 0 lowest, across the word boundary (the packing the
+/// identification code uses for every multi-word mask, `regs.rs`).
+pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUSTERS as usize] {
+    let mut masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize];
+    let width = soc.cores_per_cluster.min(32);
+    let field = (1u64 << width) - 1;
+    for (cluster, slot) in masks.iter_mut().enumerate().take(soc.clusters as usize) {
+        let shift = cluster as u32 * soc.cores_per_cluster;
+        *slot = mask.checked_shr(shift).map_or(0, |m| (m & field) as u32);
+    }
+    masks
+}
+
+/// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node
+/// has no populated OPP table.
+pub(crate) fn max_frequency_khz(pdev: &platform::Device<Core<'_>>) -> Option<u32> {
+    let node = pdev.as_ref().of_node()?;
+    let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?;
+    let mut max_hz: u64 = 0;
+    for opp in opps.children() {
+        if let Ok(hz) = opp.get_property::<u64>(c_str!("opp-hz")) {
+            max_hz = max_hz.max(hz);
+        }
+    }
+    u32::try_from(max_hz / 1000).ok().filter(|khz| *khz != 0)
+}
+
+/// The shape of the performance-state table the device tree's operating points give: states
+/// above the off state and the highest frequency in MHz (`t8122_admission::opp_table_shape`).
+/// Every cluster of a point must have the same voltage, except on a SoC with per-cluster
+/// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable
+/// OPP table.
+pub(crate) fn opp_table_shape(pdev: &platform::Device<Core<'_>>, soc: &Soc) -> Option<(u32, u32)> {
+    let node = pdev.as_ref().of_node()?;
+    let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?;
+    let mut points = KVec::new();
+    for opp in opps.children() {
+        let hz = opp.get_property::<u64>(c_str!("opp-hz")).ok()?;
+        let microvolt = opp.get_property::<KVec<u32>>(c_str!("opp-microvolt")).ok()?;
+        let mv = microvolt.first()?.div_ceil(1000);
+        if microvolt.len() != soc.clusters as usize
+            || (!soc.per_cluster_voltages && microvolt.iter().any(|uv| uv.div_ceil(1000) != mv))
+        {
+            return None;
+        }
+        // With per-cluster voltages a point is ordered by its highest cluster voltage, as the
+        // published tables are (`initdata::PStateTables`).
+        let uv = if soc.per_cluster_voltages {
+            *microvolt.iter().max()?
+        } else {
+            *microvolt.first()?
+        };
+        points.push((hz, uv), GFP_KERNEL).ok()?;
+    }
+    crate::m3::t8122_admission::opp_table_shape(points)
+}
+
+/// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the
+/// first 16 KiB of `asc` (`m3_device`); the first 16 MiB of `sgx` cover the ID block at
+/// +0xd04000, the Fender MMU block at +0xd08000 and the core masks at +0xe01500.
+fn reg_windows(soc: &Soc) -> [(&'static CStr, u64, u64); 2] {
+    [
+        (c_str!("asc"), soc.windows.asc, 0x4000),
+        (c_str!("sgx"), soc.windows.sgx, 0x100_0000),
+    ]
+}
+
+/// The `reg` cells (two address and two size cells) of the mailbox of `soc`: 16 KiB at its base.
+fn expected_mailbox_reg(soc: &Soc) -> [u32; 4] {
+    [(soc.windows.mailbox >> 32) as u32, soc.windows.mailbox as u32, 0, 0x4000]
+}
+
+/// Reserved regions of the firmware handoff, in `Resources::regions` order.
+const REGIONS: [&CStr; 6] = [
+    c_str!("ttbs"),
+    c_str!("pagetables"),
+    c_str!("handoff"),
+    c_str!("shared-l2"),
+    c_str!("fw-text"),
+    c_str!("fw-data"),
+];
+
+/// Static reserved-memory nodes that describe a handoff region when the GPU node does not list
+/// it under its own name: the bootloader fills these nodes (the UAT regions of the firmware's
+/// page-table handoff) whether or not the GPU node references them. The second-level table of
+/// the firmware's upper page-table tree ("shared-l2") is reserved this way without being in the
+/// GPU node's memory-region list.
+const REGION_NODES: [(&CStr, &CStr); 4] = [
+    (c_str!("ttbs"), c_str!("/reserved-memory/uat-ttbs")),
+    (c_str!("pagetables"), c_str!("/reserved-memory/uat-pagetables")),
+    (c_str!("handoff"), c_str!("/reserved-memory/uat-handoff")),
+    (c_str!("shared-l2"), c_str!("/reserved-memory/uat-pagetables-l2")),
+];
+
+/// Properties that describe the loaded firmware's segments on the GPU node.
+const SEGMENT_PROPS: [&CStr; 3] = [
+    c_str!("apple,m3-handoff-version"),
+    c_str!("apple,firmware-segment-vas"),
+    c_str!("apple,firmware-segment-flags"),
+];
+
+/// Power-management properties of a GPU node whose SoC takes them from the boot loader
+/// (`Soc::power_from_boot_loader`), one 32-bit cell each: the ones the T6030 device tree carries
+/// as static values, and the shader-engine target. The boot loader copies each from the ADT's GPU
+/// node (`apple,X` from `gpu-X`; `apple,se-target` from `gpu-se-tgt`).
+///
+/// `apple,idleoff-standby-timer` is not among them: the J613 ADT has no
+/// `gpu-idleoff-standby-timer`, so the boot loader adds it only when the ADT has one, and the
+/// HwConfig's default applies otherwise.
+const BOOT_LOADER_POWER: [&CStr; 40] = [
+    c_str!("apple,perf-base-pstate"),
+    c_str!("apple,min-sram-microvolt"),
+    c_str!("apple,avg-power-filter-tc-ms"),
+    c_str!("apple,avg-power-ki-only"),
+    c_str!("apple,avg-power-kp"),
+    c_str!("apple,avg-power-min-duty-cycle"),
+    c_str!("apple,avg-power-target-filter-tc"),
+    c_str!("apple,fast-die0-integral-gain"),
+    c_str!("apple,fast-die0-proportional-gain"),
+    c_str!("apple,perf-boost-ce-step"),
+    c_str!("apple,perf-boost-min-util"),
+    c_str!("apple,perf-filter-drop-threshold"),
+    c_str!("apple,perf-filter-time-constant"),
+    c_str!("apple,perf-filter-time-constant2"),
+    c_str!("apple,perf-integral-gain"),
+    c_str!("apple,perf-integral-gain2"),
+    c_str!("apple,perf-integral-min-clamp"),
+    c_str!("apple,perf-proportional-gain"),
+    c_str!("apple,perf-proportional-gain2"),
+    c_str!("apple,perf-tgt-utilization"),
+    c_str!("apple,power-sample-period"),
+    c_str!("apple,ppm-filter-time-constant-ms"),
+    c_str!("apple,ppm-ki"),
+    c_str!("apple,ppm-kp"),
+    c_str!("apple,pwr-filter-time-constant"),
+    c_str!("apple,pwr-integral-gain"),
+    c_str!("apple,pwr-integral-min-clamp"),
+    c_str!("apple,pwr-min-duty-cycle"),
+    c_str!("apple,pwr-proportional-gain"),
+    c_str!("apple,pwr-sample-period-aic-clks"),
+    c_str!("apple,se-engagement-criteria"),
+    c_str!("apple,se-filter-time-constant"),
+    c_str!("apple,se-filter-time-constant-1"),
+    c_str!("apple,se-inactive-threshold"),
+    c_str!("apple,se-ki"),
+    c_str!("apple,se-ki-1"),
+    c_str!("apple,se-kp"),
+    c_str!("apple,se-kp-1"),
+    c_str!("apple,se-reset-criteria"),
+    c_str!("apple,se-target"),
+];
+
+/// Check that the boot loader gave a GPU node of `soc` every per-machine property the runtime
+/// would otherwise take from a static device tree or a driver default: [`BOOT_LOADER_POWER`], the
+/// 64-bit `apple,fast-die0-sensor-mask`, one core and one SRAM leakage coefficient per cluster,
+/// the firmware version (three cells, not all zero), and an operating-point table of at least two
+/// entries, each with one voltage per cluster and a power, and a frequency after the first. Logs
+/// every missing or malformed property.
+fn check_boot_loader_power(dev: &device::Device, node: &of::Node, soc: &Soc) -> Result {
+    let mut bad = 0u32;
+    let mut expect = |name: &CStr, want: usize, nonzero: bool| {
+        match node.get_property::<KVec<u8>>(name) {
+            Ok(value) if value.len() == want && (!nonzero || value.iter().any(|b| *b != 0)) => {}
+            Ok(value) => {
+                dev_info!(
+                    dev,
+                    "M3: not admitted: {:?} is {} bytes, {} expected{}\n",
+                    name,
+                    value.len(),
+                    want,
+                    if nonzero { ", not all zero" } else { "" }
+                );
+                bad += 1;
+            }
+            Err(_) => {
+                dev_info!(dev, "M3: not admitted: the boot loader did not add {:?}\n", name);
+                bad += 1;
+            }
+        }
+    };
+    for name in BOOT_LOADER_POWER {
+        expect(name, 4, false);
+    }
+    expect(c_str!("apple,fast-die0-sensor-mask"), 8, false);
+    expect(c_str!("apple,core-leak-coef"), 4 * soc.clusters as usize, false);
+    expect(c_str!("apple,sram-leak-coef"), 4 * soc.clusters as usize, false);
+    expect(c_str!("apple,firmware-version"), 12, true);
+
+    let mut opps = 0u32;
+    let mut opps_ok = true;
+    match node.parse_phandle(c_str!("operating-points-v2"), 0) {
+        Some(table) => {
+            for opp in table.children() {
+                let hz = opp.get_property::<u64>(c_str!("opp-hz")).unwrap_or(0);
+                let volts = opp.get_property::<KVec<u32>>(c_str!("opp-microvolt")).map_or(0, |v| v.len());
+                let power = opp.get_property::<u32>(c_str!("opp-microwatt")).is_ok();
+                opps_ok &= volts == soc.clusters as usize && power && (opps == 0 || hz != 0);
+                opps += 1;
+            }
+        }
+        None => opps_ok = false,
+    }
+    if !opps_ok || opps < 2 {
+        dev_info!(
+            dev,
+            "M3: not admitted: the operating points are not the boot loader's ({} entries; each needs opp-hz, {} opp-microvolt cells and opp-microwatt)\n",
+            opps,
+            soc.clusters
+        );
+        bad += 1;
+    }
+    if bad != 0 {
+        return Err(EINVAL);
+    }
+    dev_info!(dev, "M3: boot loader power configuration accepted ({} operating points)\n", opps);
+    Ok(())
+}
+
+/// Whether the GPU node lists the memory region `name` in memory-region-names.
+fn lists_region(node: &of::Node, name: &CStr) -> bool {
+    node.get_property::<KVec<u8>>(c_str!("memory-region-names"))
+        .is_ok_and(|names| names.split(|b| *b == 0).any(|n| n == name.to_bytes()))
+}
+
+/// The static reserved-memory node that stands in for the handoff region `name`, if the GPU
+/// node does not list the region itself and the device tree has such a node.
+fn region_node_path(node: &of::Node, name: &CStr) -> Option<&'static CStr> {
+    if lists_region(node, name) {
+        return None;
+    }
+    REGION_NODES
+        .iter()
+        .find(|(region, _)| region.to_bytes() == name.to_bytes())
+        .map(|(_, path)| *path)
+}
+
+/// A reference to a device tree node found by path, released on drop.
+struct NodeRef(*mut bindings::device_node);
+
+impl NodeRef {
+    fn find(path: &CStr) -> Option<Self> {
+        // SAFETY: `path` is a NUL-terminated string; the lookup takes a node reference or
+        // returns NULL.
+        let np = unsafe { bindings::of_find_node_opts_by_path(path.as_char_ptr(), core::ptr::null_mut()) };
+        (!np.is_null()).then_some(NodeRef(np))
+    }
+}
+
+impl Drop for NodeRef {
+    fn drop(&mut self) {
+        // SAFETY: the reference was taken by `find`.
+        unsafe { bindings::of_node_put(self.0) };
+    }
+}
+
+/// Look up the handoff region `name` through its static reserved-memory node (see
+/// [`REGION_NODES`]). Returns None when the GPU node lists the region itself or the device tree
+/// has no such node; otherwise the region, which must be an enabled reserved-memory node the
+/// kernel reserved at boot. Returns whether the node is no-map as well.
+pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option<Result<(Resource, bool)>> {
+    let path = region_node_path(node, name)?;
+    let np = NodeRef::find(path)?;
+    // SAFETY: `np` holds a node reference for the duration of these read-only queries.
+    let (available, rmem, nomap) = unsafe {
+        (
+            bindings::of_device_is_available(np.0),
+            bindings::of_reserved_mem_lookup(np.0),
+            bindings::of_property_read_bool(np.0, c_str!("no-map").as_char_ptr()),
+        )
+    };
+    if !available || rmem.is_null() {
+        return Some(Err(EINVAL));
+    }
+    // SAFETY: the reserved-memory table entry lives as long as the kernel.
+    let (base, size) = unsafe { ((*rmem).base, (*rmem).size) };
+    if base == 0 || size == 0 || base.checked_add(size).is_none_or(|end| end > 1 << 42) {
+        return Some(Err(EINVAL));
+    }
+    let raw = bindings::resource {
+        start: base,
+        end: base + size - 1,
+        flags: bindings::IORESOURCE_MEM as _,
+        ..Default::default()
+    };
+    // SAFETY: Resource is repr(transparent) over Opaque<resource>. The descriptor is owned and
+    // has no pointers into temporary data; its range is checked above.
+    Some(Ok((unsafe { core::mem::transmute::<bindings::resource, Resource>(raw) }, nomap)))
+}
+
+/// Admit a GPU of `soc` described by the device tree, either statically or by a runtime overlay.
+///
+/// The device must be a GPU node of `soc` with the ASC and SGX windows at their addresses
+/// (at least as large as the runtime maps), the GPU coprocessor mailbox, the six handoff regions
+/// by name (from the reserved-memory registry or no-map overlay nodes), and the firmware segment
+/// description. Every refusal is logged.
+pub(crate) fn admit(pdev: &platform::Device<Core<'_>>, soc: &Soc) -> Result<Resources> {
+    let dev = pdev.as_ref();
+    let refuse = |what: &str, err: Error| {
+        dev_info!(dev, "M3: not admitted: {}\n", what);
+        err
+    };
+    let node = dev.of_node().ok_or(ENODEV)?;
+
+    if !board_is(soc.board.as_bytes()) {
+        dev_info!(dev, "M3: not admitted: not a {} board\n", soc.name);
+        return Err(ENODEV);
+    }
+    let compatible: KVec<u8> = node.get_property(c_str!("compatible"))?;
+    if !compatible.split(|b| *b == 0).any(|s| s == soc.gpu.as_bytes()) {
+        dev_info!(dev, "M3: not admitted: GPU node is not {}\n", soc.gpu);
+        return Err(ENODEV);
+    }
+    // The runtime runs the firmware that the bootloader loaded, in place: without the
+    // bootloader's description of its segments there is nothing to admit.
+    if !lists_region(&node, c_str!("fw-text"))
+        || !lists_region(&node, c_str!("fw-data"))
+        || SEGMENT_PROPS.iter().any(|p| node.get_property::<KVec<u8>>(p).is_err())
+    {
+        return Err(refuse(
+            "the device tree does not describe the loaded GPU firmware segments (fw-text/fw-data regions and apple,firmware-segment-* properties); the bootloader predates this",
+            ENODEV,
+        ));
+    }
+    dev_info!(dev, "M3: board compatible accepted\n");
+    if soc.power_from_boot_loader {
+        check_boot_loader_power(dev, &node, soc)?;
+    }
+
+    for name in REGIONS {
+        if lists_region(&node, name) {
+            continue;
+        }
+        match region_node_path(&node, name) {
+            Some(path) if NodeRef::find(path).is_some() => {
+                dev_info!(dev, "M3: {:?} region: static node {:?}\n", name, path)
+            }
+            _ => {
+                dev_info!(dev, "M3: not admitted: no {:?} memory region\n", name);
+                return Err(EINVAL);
+            }
+        }
+    }
+    dev_info!(dev, "M3: memory region list accepted\n");
+
+    for (name, base, min_size) in reg_windows(soc) {
+        let res = pdev
+            .resource_by_name(name)
+            .ok_or_else(|| refuse("missing register window", EINVAL))?;
+        if res.start() != base || res.size() < min_size {
+            dev_info!(
+                dev,
+                "M3: not admitted: {:?} window {:#x}+{:#x}, need {:#x}+{:#x} or larger\n",
+                name,
+                res.start(),
+                res.size(),
+                base,
+                min_size
+            );
+            return Err(EINVAL);
+        }
+    }
+    dev_info!(dev, "M3: register resources accepted\n");
+
+    let mboxes: KVec<u32> = node.get_property(c_str!("mboxes"))?;
+    let mbox = (mboxes.len() == 1)
+        .then(|| node.parse_phandle(c_str!("mboxes"), 0))
+        .flatten()
+        .ok_or_else(|| refuse("expected exactly one mailbox", EINVAL))?;
+    let compat: KVec<u8> = mbox.get_property(c_str!("compatible"))?;
+    let mbox_reg: KVec<u32> = mbox.get_property(c_str!("reg"))?;
+    let irq_names: KVec<u8> = mbox.get_property(c_str!("interrupt-names"))?;
+    let irqs: KVec<u32> = mbox.get_property(c_str!("interrupts"))?;
+    let cells: u32 = mbox.get_property(c_str!("#mbox-cells"))?;
+    if !soc.mailbox_compatibles.iter().any(|c| *c == compat.as_slice())
+        || mbox_reg.as_slice() != expected_mailbox_reg(soc)
+        || cells != 0
+        || irq_names.as_slice() != b"send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0"
+        || irqs.as_slice() != soc.mailbox_interrupts
+    {
+        return Err(refuse("unexpected GPU mailbox description", EINVAL));
+    }
+    dev_info!(dev, "M3: mailbox resource accepted\n");
+
+    let mut regions = [Region { base: 0, size: 0 }; 6];
+    for (i, name) in REGIONS.iter().enumerate() {
+        let res = reserved_resource(&node, name).map_err(|e| {
+            dev_info!(dev, "M3: not admitted: {:?} region unusable ({:?})\n", name, e);
+            e
+        })?;
+        regions[i] = Region {
+            base: res.start(),
+            size: res.size(),
+        };
+    }
+    dev_info!(dev, "M3: reserved resources {:?}\n", regions);
+
+    let version: u32 = node
+        .get_property(c_str!("apple,m3-handoff-version"))
+        .map_err(|e| refuse("no firmware handoff description", e))?;
+    let vas: KVec<u64> = node.get_property(c_str!("apple,firmware-segment-vas"))?;
+    let flags: KVec<u32> = node.get_property(c_str!("apple,firmware-segment-flags"))?;
+    Resources::validate(
+        version,
+        regions,
+        vas.as_slice().try_into().map_err(|_| EINVAL)?,
+        flags.as_slice().try_into().map_err(|_| EINVAL)?,
+    )
+    .map_err(|e| {
+        dev_info!(dev, "M3: not admitted: handoff description rejected ({:?})\n", e);
+        EINVAL
+    })
+}
+
+/// Whether the GPU node links to a PMP instance (`apple,pmp`) through which the GPU power vote
+/// is cast. Without one, GPU power is left to the power domain.
+pub(crate) fn has_pmp_link(pdev: &platform::Device<Core<'_>>) -> bool {
+    pdev.as_ref()
+        .of_node()
+        .is_some_and(|node| node.get_property::<u32>(c_str!("apple,pmp")).is_ok())
+}
+
+pub(crate) use crate::m3::m3_firmware::{KnownImage, KNOWN_IMAGES_T8122};
+use crate::m3::m3_firmware::image_info;
+
+fn sha256(data: &[u8]) -> [u8; 32] {
+    let mut digest = [0u8; 32];
+    // SAFETY: `data` and `digest` are valid for the duration of the synchronous call.
+    unsafe { bindings::sha256(data.as_ptr(), data.len(), digest.as_mut_ptr()) };
+    digest
+}
+
+/// Lower-case hex of `bytes`, for logs.
+fn hex<const N: usize>(bytes: &[u8]) -> [u8; N] {
+    const DIGITS: &[u8; 16] = b"0123456789abcdef";
+    let mut out = [b'0'; N];
+    for (i, b) in bytes.iter().take(N / 2).enumerate() {
+        out[2 * i] = DIGITS[(b >> 4) as usize];
+        out[2 * i + 1] = DIGITS[(b & 0xf) as usize];
+    }
+    out
+}
+
+pub(crate) fn identify(
+    dev: &device::Device,
+    text: &[u8],
+    stkg_sha256: &[u8; 32],
+    images: &'static [KnownImage],
+) -> Option<&'static KnownImage> {
+    let info = image_info(text);
+    let masked_sha256 = info.as_ref().and_then(|info| {
+        let mut copy = KVec::new();
+        copy.extend_from_slice(text, GFP_KERNEL).ok()?;
+        copy.get_mut(info.patchbay.clone())?.fill(0);
+        Some(sha256(&copy))
+    });
+    let uuid = info.as_ref().map(|info| info.uuid);
+    let uuid_hex = hex::<32>(&uuid.unwrap_or([0; 16]));
+    let stkg_hex = hex::<64>(stkg_sha256);
+    let masked_hex = hex::<64>(&masked_sha256.unwrap_or([0; 32]));
+    fn as_str(b: &[u8]) -> &str {
+        core::str::from_utf8(b).unwrap_or("?")
+    }
+    dev_info!(
+        dev,
+        "M3: loaded GPU firmware: uuid {} (header {}), text sha256 {} (STKG zeroed), {} (patchbay zeroed)\n",
+        if uuid.is_some() { as_str(&uuid_hex) } else { "none" },
+        if info.as_ref().is_some_and(|i| i.text_size == text.len()) { "ok" } else { "missing or unexpected" },
+        as_str(&stkg_hex),
+        if masked_sha256.is_some() { as_str(&masked_hex) } else { "n/a" }
+    );
+    let image = images.iter().find(|image| image.accepts_runtime(uuid, stkg_sha256));
+    match image {
+        Some(image) => dev_info!(dev, "M3: GPU firmware identified as {}\n", image.name),
+        None => dev_err!(dev, "M3: loaded GPU firmware is not a known image\n"),
+    }
+    image
+}
+
+/// Whether the reserved-memory node behind the GPU memory region `name` is no-map.
+///
+/// A region without no-map is part of the kernel's linear map, which maps it write-back; any
+/// other CPU mapping of it must use the same memory type, because mismatched aliases of the same
+/// memory are not allowed. Only no-map regions may be mapped write-combined. Returns true when
+/// the region cannot be looked up, which keeps the caller's own choice.
+pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool {
+    if let Some(found) = static_region(node, name) {
+        return found.map_or(true, |(_, nomap)| nomap);
+    }
+    let Ok(names) = node.get_property::<KVec<u8>>(c_str!("memory-region-names")) else {
+        return true;
+    };
+    let Some(index) = names.split(|b| *b == 0).position(|n| n == name.to_bytes()) else {
+        return true;
+    };
+    let Some(region) = node.parse_phandle(c_str!("memory-region"), index) else {
+        return true;
+    };
+    region.get_property::<KVec<u8>>(c_str!("no-map")).is_ok()
+}
+
+/// Whether the M3 render node is registered (`asahi.m3_expose`).
+///
+/// Userspace drivers pick up any registered render node. Until the M3 runtime is validated on a
+/// board, a render node there would let a stock userspace driver submit work the board may not
+/// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on.
+pub(crate) fn expose_render_node(soc: &Soc) -> bool {
+    match crate::m3::m3_params::expose() {
+        Some(expose) => expose,
+        None => runtime_validated_board(soc),
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_channel_format.rs b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs
new file mode 100644
index 0000000000000..45eebda112b73
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Bounded firmware channel records. Publication and retirement belong to Config.
+
+use super::m3_init_layout::Error;
+
+pub(crate) const CONTROL_BYTES: usize = 64;
+pub(crate) const RECEIVE_BYTES: usize = 72;
+pub(crate) const SLOTS: u32 = 256;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum Action {
+    Initialize,
+    Register {
+        pipe: u32,
+        engine: u32,
+    },
+    Idle {
+        enabled: bool,
+    },
+    Release {
+        group: u8,
+        index: u8,
+        weight: u8,
+        address: u64,
+    },
+}
+
+impl Action {
+    /// The first two actions precede the first generation-bearing publication.
+    pub(crate) fn generation(self, published: u32, consumed: u32) -> Result<u32, Error> {
+        match self {
+            Self::Initialize | Self::Register { .. } => {
+                if published != 0 || consumed != 0 {
+                    return Err(Error::Address);
+                }
+                Ok(0)
+            }
+            _ => published.max(consumed).checked_add(1).ok_or(Error::Address),
+        }
+    }
+
+    pub(crate) fn encode(self, output: &mut [u8]) -> Result<(), Error> {
+        let mut words = [0u32; 3];
+        let mut release = None;
+        match self {
+            Self::Initialize => words[0] = 0x16,
+            Self::Register { pipe, engine } => {
+                if pipe >= 3 || !(1..=2).contains(&engine) {
+                    return Err(Error::Address);
+                }
+                words = [0x1b, pipe, engine];
+            }
+            Self::Idle { enabled } => words = [0x0a, enabled.into(), 0],
+            Self::Release {
+                group,
+                index,
+                weight,
+                address,
+            } => {
+                if index == 255
+                    || weight == 255
+                    || address < 0xffff_fc00_0000_0000
+                    || address & 7 != 0
+                {
+                    return Err(Error::Address);
+                }
+                words[0] = 0x14;
+                words[2] = u32::from_le_bytes([group, index, weight, 4]);
+                release = Some(address);
+            }
+        }
+        let bytes = output.get_mut(..CONTROL_BYTES).ok_or(Error::Size)?;
+        bytes.fill(0);
+        for (index, word) in words.iter().enumerate() {
+            bytes[index * 4..index * 4 + 4].copy_from_slice(&word.to_le_bytes());
+        }
+        if let Some(address) = release {
+            bytes[12..20].copy_from_slice(&address.to_le_bytes());
+        }
+        Ok(())
+    }
+}
+
+/// An empty ring is required before publishing a synchronous control transaction.
+/// Both consumers must retire it before the next control action may be published.
+pub(crate) fn idle_cursor(cursors: [u32; 3]) -> Result<u32, Error> {
+    if cursors.iter().any(|value| *value >= SLOTS) {
+        return Err(Error::Bounds);
+    }
+    if cursors[0] != cursors[2] || cursors[1] != cursors[2] {
+        return Err(Error::Address);
+    }
+    Ok(cursors[2])
+}
+
+pub(crate) fn payload_offset(thread: usize, slot: u32) -> Result<usize, Error> {
+    if thread >= 9 {
+        return Err(Error::Bounds);
+    }
+    Ok((thread * 256 + (slot & 255) as usize) * 0xd8)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn initialization_records_have_only_their_defined_words() {
+        for action in [Action::Initialize, Action::Register { pipe: 2, engine: 1 }] {
+            let mut output = [0xa5; CONTROL_BYTES + 8];
+            action.encode(&mut output).unwrap();
+            assert_eq!(&output[12..CONTROL_BYTES], &[0; CONTROL_BYTES - 12]);
+            assert_eq!(&output[CONTROL_BYTES..], &[0xa5; 8]);
+            assert_eq!(action.generation(0, 0), Ok(0));
+            assert!(action.generation(1, 0).is_err());
+        }
+    }
+
+    #[test]
+    fn invalid_actions_preserve_the_entire_output() {
+        for action in [
+            Action::Register { pipe: 3, engine: 1 },
+            Action::Register { pipe: 0, engine: 0 },
+            Action::Release {
+                group: 0,
+                index: 255,
+                weight: 1,
+                address: 0xffff_fc21_0000_0000,
+            },
+            Action::Release {
+                group: 0,
+                index: 0,
+                weight: 1,
+                address: 8,
+            },
+        ] {
+            let mut output = [0xa5; CONTROL_BYTES];
+            assert!(action.encode(&mut output).is_err());
+            assert_eq!(output, [0xa5; CONTROL_BYTES]);
+        }
+    }
+
+    #[test]
+    fn generations_follow_consumption_and_never_wrap() {
+        let action = Action::Idle { enabled: true };
+        assert_eq!(action.generation(3, 5), Ok(6));
+        assert_eq!(action.generation(5, 3), Ok(6));
+        assert!(action.generation(u32::MAX, 0).is_err());
+        assert!(action.generation(0, u32::MAX).is_err());
+        let mut output = [0; CONTROL_BYTES];
+        action.encode(&mut output).unwrap();
+        assert_eq!(&output[..8], &[10, 0, 0, 0, 1, 0, 0, 0]);
+    }
+
+    #[test]
+    fn both_consumers_and_all_indices_are_checked() {
+        assert_eq!(idle_cursor([255, 255, 255]), Ok(255));
+        for cursors in [[0, 1, 1], [1, 0, 1], [256, 256, 256]] {
+            assert!(idle_cursor(cursors).is_err());
+        }
+        assert_eq!(payload_offset(8, 0x101), Ok((8 * 256 + 1) * 0xd8));
+        assert!(payload_offset(9, 0).is_err());
+    }
+
+    #[test]
+    fn release_uses_the_packed_owned_pointer() {
+        let action = Action::Release {
+            group: 1,
+            index: 2,
+            weight: 3,
+            address: 0xffff_fc21_0400_0000,
+        };
+        let mut output = [0; CONTROL_BYTES];
+        action.encode(&mut output).unwrap();
+        assert_eq!(&output[8..12], &[1, 2, 3, 4]);
+        assert_eq!(
+            u64::from_le_bytes(output[12..20].try_into().unwrap()),
+            0xffff_fc21_0400_0000
+        );
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs
new file mode 100644
index 0000000000000..c1a07d61120d1
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_client.rs
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Safety checks for clients of the M3 (G15) render node.
+//!
+//! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for
+//! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2
+//! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client
+//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields
+//! of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks GET_PARAMS
+//! for at least the structure up to them; older userspace asks for less. On G15 a file therefore
+//! has to ask for those fields before it can create a VM. Every path to the GPU needs a VM, so a
+//! refused client can still query the parameters but never reaches the hardware.
+
+use core::sync::atomic::{
+    AtomicU32,
+    Ordering, //
+};
+
+use kernel::{
+    prelude::*,
+    uapi,
+};
+
+use crate::{
+    m3::agx_uapi::{
+        GpuAccess,
+        GpuAddressSpace, //
+    },
+    mmu,
+};
+
+/// GET_PARAMS size an AGX3-aware client asks for: the global parameter structure up to the end
+/// of `gpu_hal_generation`. The optional tail fields that follow it are not part of the check,
+/// so growing the structure does not refuse userspace built against the shorter one.
+const AGX3_PARAMS_SIZE: u64 =
+    (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation)
+        + core::mem::size_of::<u32>()) as u64;
+const _: () = assert!(AGX3_PARAMS_SIZE == super::m3_profile::PARAMS_REQUIRED);
+
+/// Refused clients that are logged per boot; later refusals only count.
+const REFUSALS_LOGGED: u32 = 16;
+static REFUSALS: AtomicU32 = AtomicU32::new(0);
+
+/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for at least
+/// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass
+/// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for.
+pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result {
+    admit_vm_for(device, super::m3_profile::Profile::Legacy, size, flags)
+}
+
+pub(crate) fn admit_vm_for(
+    device: &kernel::device::Device,
+    profile: super::m3_profile::Profile,
+    size: u64,
+    flags: u32,
+) -> Result {
+    {
+        // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`):
+        // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime.
+        if profile.vm_acknowledged(size, flags) {
+            return Ok(());
+        }
+        let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1);
+        if refusals <= REFUSALS_LOGGED {
+            let task = kernel::current!();
+            let comm = task_comm(task);
+            let len = comm.iter().position(|&b| b == 0).unwrap_or(comm.len());
+            let name = core::str::from_utf8(&comm[..len]).unwrap_or("?");
+            let last = if refusals == REFUSALS_LOGGED {
+                " (further refusals are not logged)"
+            } else {
+                ""
+            };
+            dev_info!(
+                device,
+                "M3: refusing a VM for {}[{}]: GET_PARAMS bytes={}/{} HAL={} flags={:#x}{}\n",
+                name,
+                task.group_leader().pid(),
+                size,
+                AGX3_PARAMS_SIZE,
+                profile.hal(),
+                flags,
+                last
+            );
+        }
+        Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)))
+    }
+}
+
+/// The name of `task`, for log lines only.
+pub(crate) fn task_comm(task: &kernel::task::Task) -> [u8; 16] {
+    let mut name = [0u8; 16];
+    // SAFETY: `comm` is a fixed-size array inside the live task. It is read byte by byte with
+    // volatile loads because a concurrent rename may change it; a torn name is only printed.
+    unsafe {
+        let comm = core::ptr::addr_of!((*task.as_ptr()).comm).cast::<u8>();
+        for (i, byte) in name.iter_mut().enumerate() {
+            *byte = comm.add(i).read_volatile();
+        }
+    }
+    name
+}
+
+/// Bytes at the start of a compute control stream whose launch records are checked.
+const LAUNCH_WINDOW: usize = 4096;
+/// Bytes that must be readable at a launch's program pointer. The pointer is 64-byte aligned,
+/// so this never crosses a page.
+const PROGRAM_BYTES: u64 = 64;
+/// CDM record types (bits 31:29 of a record's first word).
+const CDM_LAUNCH: u32 = 0;
+const CDM_BARRIER: u32 = 3;
+
+/// Refuse a compute command whose control stream starts with a launch the GPU cannot fetch.
+///
+/// Walks the records at the start of the stream: launches (40 bytes direct, 36 or 24 bytes
+/// indirect, told apart by bits 28:27) and barriers (4 bytes). The walk ends at the first link,
+/// return, terminator or unknown record, at the end of the stream and after `LAUNCH_WINDOW`
+/// bytes. Each launch names its program at `(word1[31:22] << 38) | (word2 << 6)`; that address,
+/// and the one read with only word2's low 26 bits when VA[37:32] is set, must be mapped
+/// readable in the submitting VM, outside the driver's own mappings, or the command is refused
+/// with EINVAL before it is queued. Without the check the GPU would take a translation fault on
+/// the fetch, which the M3 backends cannot recover from.
+///
+/// This catches malformed or stale pointers, not a hostile client: userspace can rewrite the
+/// stream after the check.
+pub(crate) fn check_compute_launches<A: GpuAddressSpace>(
+    vm: &mmu::Vm,
+    space: &A,
+    base: u64,
+    end: u64,
+) -> Result {
+    let len = usize::try_from(end.checked_sub(base).ok_or(EINVAL)?)
+        .unwrap_or(usize::MAX)
+        .min(LAUNCH_WINDOW);
+    let mut stream = KVec::from_elem(0u8, len, GFP_KERNEL)?;
+    if let Err(error) = read_through_cache(vm, base, &mut stream) {
+        pr_err!(
+            "M3 compute rejected: control stream {:#x} cannot be read ({:?})\n",
+            base,
+            error
+        );
+        return Err(EINVAL);
+    }
+    let word = |offset: usize| {
+        u32::from_le_bytes([
+            stream[offset],
+            stream[offset + 1],
+            stream[offset + 2],
+            stream[offset + 3],
+        ])
+    };
+    let mut offset = 0;
+    let mut launch = 0u32;
+    while offset + 4 <= len {
+        let head = word(offset);
+        match head >> 29 {
+            CDM_LAUNCH => {
+                if offset + 12 > len {
+                    break;
+                }
+                let high = u64::from(word(offset + 4) >> 22) << 38;
+                let low = u64::from(word(offset + 8));
+                // Whether the GPU takes VA[37:32] from word2's top six bits is not
+                // established, so the program must be mapped under both readings. They
+                // agree whenever those bits are clear, as in every working launch.
+                for program in [high | (low << 6), high | ((low & 0x3ff_ffff) << 6)] {
+                    if !space.covers(program, PROGRAM_BYTES, GpuAccess::Read) {
+                        pr_err!(
+                            "M3 compute rejected: launch {} at {:#x} names program {:#x}, which is not mapped readable in this VM\n",
+                            launch,
+                            base + offset as u64,
+                            program
+                        );
+                        return Err(EINVAL);
+                    }
+                }
+                launch += 1;
+                offset += match (head >> 27) & 3 {
+                    0 => 40,
+                    1 => 36,
+                    2 => 24,
+                    _ => break,
+                };
+            }
+            CDM_BARRIER => offset += 4,
+            _ => break,
+        }
+    }
+    Ok(())
+}
+
+/// Read `out.len()` bytes at `iova` in `vm` through the kernel's linear map.
+///
+/// Userspace normally writes command streams through a write-combined mapping, which does not
+/// update cache lines the kernel's cacheable alias may hold from an earlier read or a
+/// speculative fetch. Each line is cleaned and invalidated before it is read, so the bytes
+/// come from memory, as the GPU will fetch them. Cleaning never discards data: a dirty line from
+/// a cached mapping is written back first.
+///
+/// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the
+/// pages cannot be unmapped and freed while they are cleaned and read.
+fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result {
+    // 7.2's `Vm::read_user` cleans and invalidates each line to the point of coherency before
+    // the copy, under the VM execution lock.
+    vm.read_user(iova, out)
+}
+
+/// Clean and invalidate the data cache lines covering `start..start + len` to the point of
+/// coherency, completing before later loads.
+fn clean_invalidate(start: usize, len: usize) {
+    let ctr: u64;
+    // SAFETY: CTR_EL0 is readable at EL1 and has no side effects.
+    unsafe {
+        core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, options(nomem, nostack, preserves_flags));
+    }
+    // DminLine: log2 of the smallest data cache line, in 4-byte words.
+    let line = 4usize << ((ctr >> 16) & 0xf);
+    let end = start.saturating_add(len);
+    let mut address = start & !(line - 1);
+    // SAFETY: the range lies inside one page of the linear map that the caller has mapped.
+    // Clean-and-invalidate writes dirty lines back before dropping them, so no data is lost.
+    unsafe {
+        core::arch::asm!("dsb sy", options(nostack, preserves_flags));
+        while address < end {
+            core::arch::asm!("dc civac, {address}", address = in(reg) address, options(nostack, preserves_flags));
+            address += line;
+        }
+        core::arch::asm!("dsb sy", options(nostack, preserves_flags));
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs
new file mode 100644
index 0000000000000..387aaeca1b6c6
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Completion work owned only by the M3 runtime.
+
+use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}};
+use crate::m3::{m3_drm::Shared, m3_rtkit::State};
+
+#[pin_data]
+pub(crate) struct Completion {
+    shared: Shared,
+    #[pin]
+    work: Work<Completion>,
+}
+impl Completion {
+    pub(crate) fn new(shared: Shared) -> Result<Arc<Self>> {
+        Arc::pin_init(try_pin_init!(Self {
+            shared,
+            work <- new_work!("M3 completion"),
+        }), GFP_KERNEL)
+    }
+}
+impl_has_work! { impl HasWork<Completion> for Completion { self.work } }
+impl WorkItem for Completion {
+    type Pointer = Arc<Self>;
+    fn run(owner: Arc<Self>) {
+        use crate::m3::m3_runtime::Service;
+        // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep.
+        // Check immediately on IRQ, with backoff for trailing coherent witnesses.
+        let mut policy = super::m3_completion_policy::WaitPolicy::new();
+        loop {
+            let (outcome, events) = {
+                let mut guard = owner.shared.lock();
+                let Some(runtime) = Option::as_mut(&mut *guard) else { return; };
+                let outcome=runtime.service_job();
+                let events=runtime.events();
+                // Preserve the root-only CPU-overlap diagnostic: disabled waits retain
+                // the runtime lock so VM/timestamp preparation cannot overlap the GPU.
+                if let Service::Waiting(messages)=outcome {
+                    if !events.cpu_overlap() {
+                        events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
+                        continue;
+                    }
+                }
+                (outcome,events)
+            };
+            match outcome {
+                Service::Idle => break,
+                Service::Progress => policy.progress(),
+                Service::Waiting(messages) => {
+                    // IRQs and timer expiry alone are not retirement progress.
+                    // Keep backing off until service publishes or retires work.
+                    events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
+                }
+            }
+        }
+    }
+}
+pub(crate) fn queue(state: &State) {
+    if let Some(work) = state.completion.as_ref() {
+        let _ = workqueue::system_highpri().enqueue(work.clone());
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs
new file mode 100644
index 0000000000000..9c0da8c3246e3
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Bounded fallback for witnesses which can become visible after the last IRQ.
+//!
+//! Notifications request a fresh snapshot, but do not prove witness progress.
+//! Only publication or retirement resets this policy. In particular, neither a
+//! timer wake nor a stream of early firmware notifications renews a spin budget.
+
+pub(crate) struct WaitPolicy {
+    next_us: u64,
+}
+
+impl WaitPolicy {
+    pub(crate) const fn new() -> Self {
+        Self { next_us: 25 }
+    }
+
+    pub(crate) fn progress(&mut self) {
+        self.next_us = 25;
+    }
+
+    /// No deliberate busy delay: IRQs still wake each sleep immediately.
+    /// The caller supplies the root-selected ceiling (25..=1000 us).
+    pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 {
+        let wait = self.next_us.min(ceiling_us.clamp(25, 1000));
+        self.next_us = (wait * 2).min(1000);
+        wait
+    }
+}
+
+#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))]
+mod tests {
+    use super::WaitPolicy;
+
+    #[test]
+    fn no_progress_backs_off_and_stays_bounded() {
+        let mut policy = WaitPolicy::new();
+        for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] {
+            assert_eq!(policy.next_wait_us(1000), expected);
+        }
+        for _ in 0..100_000 {
+            assert_eq!(policy.next_wait_us(1000), 1000);
+        }
+    }
+
+    #[test]
+    fn publication_or_retirement_restarts_fast_detection() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        policy.progress();
+        assert_eq!(policy.next_wait_us(1000), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+    }
+
+    #[test]
+    fn live_ceiling_changes_apply_to_the_next_wait() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        assert_eq!(policy.next_wait_us(25), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+        assert_eq!(policy.next_wait_us(100), 100);
+        assert_eq!(policy.next_wait_us(100), 100);
+    }
+
+    #[test]
+    fn fallback_keeps_timeout_checks_frequent_without_any_irq() {
+        let mut policy = WaitPolicy::new();
+        let mut elapsed_us = 0;
+        while elapsed_us < 2_000_000 {
+            let wait = policy.next_wait_us(1000);
+            assert!(wait <= 1000);
+            elapsed_us += wait;
+        }
+        // This bounds the requested delay, not scheduler dispatch latency.
+        assert!(elapsed_us < 2_001_000);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs
new file mode 100644
index 0000000000000..b6e9444ed5449
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM.
+use kernel::prelude::*;
+use crate::m3::m3_queue_layout as queue;
+use crate::m3::m3_state_layout as state;
+use crate::{m3::m3_compute_layout as command, m3::m3_compute_sequence as microsequence,
+    m3::m3_pool_layout::GpuRegion, m3::m3_init_layout::Region};
+use crate::{driver,mmu,m3::agx_memory::{self},m3::m3_memory::Buffer,m3::m3_compute_storage as storage};
+use crate::m3::m3_timeline as timeline;
+use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS,
+    TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping};
+const UMA_POOL:crate::m3::m3_pool::Owners=crate::m3::m3_pool::Owners {
+    backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE,
+    manager:storage::POOL,counter:storage::COUNTER,table_index:2,
+};
+// Retain at most four VM views, including the active one, as the render
+// graph does. Game and compositor compute jobs otherwise remap the whole
+// private arena every time they alternate. Each entry pins its VM binding.
+struct CachedViews {
+    command: mmu::KernelMapping,
+    views: KVec<(usize, mmu::KernelMapping)>,
+    // Release cached mappings before returning their ASID lease.
+    binding: mmu::VmBind,
+}
+pub(crate) struct Compute {
+    _client_command: mmu::KernelMapping,
+    objects: KVec<Buffer>,
+    command_bytes: KVec<u8>,
+    sequence_bytes: KVec<u8>,
+    _binding: mmu::VmBind,
+    stats: Region,
+    sequence: u64,
+    last_progress_ns: i64,
+    head: u16,
+    cached_views: KVec<CachedViews>,
+    slot: usize,
+    batch_count: usize,
+    checkpoint: (u64,u16,usize),
+    /// Firmware-only barrier storage, allocated only for mixed publications.
+    barriers: Option<Buffer>,
+    /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`).
+    registers: command::RegisterSet,
+    profile: super::m3_profile::Profile,
+    /// Last: drops after the command aliases.
+    _lower_tlbi: crate::m3::m3_memory::LowerTlbi,
+}
+impl Compute {
+    pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm,
+        stats: Region, control: crate::m3::agx_compute::Control,
+        registers: command::RegisterSet, profile: super::m3_profile::Profile) -> Result<Self> {
+        let lower_tlbi=crate::m3::m3_memory::LowerTlbi;
+        let binding=uat.bind(vm)?;
+        let mut objects=KVec::new();
+        let mut client_command=None;
+        let mut command_bytes = KVec::new();
+        command_bytes.resize(if profile == super::m3_profile::Profile::Extended {
+            command::EXTENDED_SIZE
+        } else { command::SIZE }, 0u8, GFP_KERNEL)?;
+        let mut sequence_bytes = KVec::new();
+        sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?;
+        // A private compute aperture, disjoint from the retained render graph.
+        // Address-bearing graph fields are constructed from their live owners.
+        for index in 0..storage::COUNT {
+            let a=storage::batch_allocation_for(profile,index).map_err(|_|EINVAL)?;
+            let mut b=match a.mapping {
+                Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?,
+                Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?,
+                Mapping::FirmwareGpuShared=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,true)?,
+                Mapping::FirmwareOnly=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,false)?,
+            };
+            if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);}
+            objects.push(b,GFP_KERNEL)?;
+        }
+        Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0,
+            registers, profile, binding.generation())?;
+        Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?;
+        if profile == super::m3_profile::Profile::Extended {
+            UMA_POOL.initialize_extended(&mut objects, 1, true)?;
+        } else { UMA_POOL.initialize(&mut objects)?; }
+        Self::initialize_state(&mut objects, binding.slot(), profile)?;
+        // Queue references are firmware VAs from live allocations. In contrast
+        // to render's unpublished seed entries, compute dispatches entry zero
+        // immediately, so its initial producer cursor remains one.
+        let address = |index: usize| queue::FirmwareVa::new(objects[index].va()).map_err(|_| EINVAL);
+        let info = queue::QueueInfo { state: address(storage::QUEUE_STATE)?, ring: address(storage::RING)?,
+            job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?,
+            context: address(GPU_CONTEXT)?, uuid: 0xc10000 };
+        let command = address(storage::COMMAND)?;
+        let mut bytes = KVec::new();
+        bytes.resize(if profile == super::m3_profile::Profile::Extended {
+            queue::QueueInfo::EXTENDED_SIZE
+        } else { queue::INFO_SIZE }, 0u8, GFP_KERNEL)?;
+        if profile == super::m3_profile::Profile::Extended {
+            info.encode_extended(address(storage::QUEUE)?, &mut bytes).map_err(|_| EINVAL)?;
+        } else { info.encode(&mut bytes).map_err(|_| EINVAL)?; }
+        objects[storage::QUEUE].write(0, &bytes)?;
+        let mut state = [0; queue::STATE_SIZE];
+        queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 }
+            .encode(&mut state).map_err(|_| EINVAL)?;
+        objects[storage::QUEUE_STATE].write(0, &state)?;
+        for index in 0..queue::QUALIFIED_CAPACITY {
+            let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &[command]).map_err(|_| EINVAL)?;
+            objects[storage::RING].write(index as usize * 8, &entry)?;
+        }
+        dev_info!(dev.as_ref(),"M3 compute graph: context={} queue={:#x} command={:#x} low={:#x} cdm={:#x}..{:#x} pool={:#x}\n",
+            binding.slot(),objects[storage::QUEUE].va(),objects[storage::COMMAND].va(),
+            objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va());
+        // The M3 compute stream uses absolute IOTO addresses and does not
+        // include the USC_EXEC_BASE_CP register.
+        // Diagnostic only: read no further than the stream, and never fail the job over it.
+        let mut stream=[0u8;48];
+        let len=core::cmp::min(control.end.saturating_sub(control.base),stream.len() as u64) as usize;
+        if vm.read_bytes(control.base,&mut stream[..len]).is_ok() {
+            crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",&stream[..len]);
+        }
+        agx_memory::publish();
+        // Finish the write-combining page-table stores before broadcasting
+        // the invalidation.
+        crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();
+        unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))};
+        Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi})
+    }
+    fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32,
+        control: crate::m3::agx_compute::Control, sequence: u64, slot: usize,
+        register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result {
+        let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL);
+        let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL);
+        let preemption = &objects[PREEMPTION];
+        let value = command::Command {
+            context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?,
+            preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation_for(profile,PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?,
+            cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?,
+            cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?,
+            gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?,
+            microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size:
+                if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 }
+                else { microsequence::LENGTH as u32 },
+            stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?,
+            stamp_value: timeline::stamp(0xc1000000,sequence),
+            timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?],
+            pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false,
+            register_set,
+        };
+        value.encode_with_usc(profile, generation, 0,
+            if profile == super::m3_profile::Profile::Extended { control.usc_base } else { 0 }, bytes).map_err(|_| EINVAL)?;
+        objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes)
+    }
+    fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region,
+        context: u32, stamp_value: u32, slot: usize, profile: super::m3_profile::Profile,
+        queue_sequence: u64) -> Result {
+        let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL);
+        let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL);
+        let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation_for(profile,owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL);
+        let value = microsequence::Sequence {
+            command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?,
+            queue: fw(storage::QUEUE)?, context,
+            generation: if profile == super::m3_profile::Profile::Extended { 0 } else { 1 }, pool: fw(storage::POOL)?,
+            scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?,
+                fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?],
+            fw_stamp: fw(storage::FW_STAMP)?, stamp_value,
+        };
+        if profile == super::m3_profile::Profile::Extended {
+            let auxiliary = command::GpuVa::new(objects[TAIL_SCRATCH].gpu_va()? + offset(TAIL_SCRATCH)? as u64).map_err(|_| EINVAL)?;
+            let context_store = GpuRegion::new(objects[MICROSEQUENCE_SCRATCH[1]].gpu_va()?
+                + offset(MICROSEQUENCE_SCRATCH[1])? as u64, 0x40000).map_err(|_| EINVAL)?;
+            value.encode_extended(auxiliary, context_store, queue_sequence, bytes).map_err(|_| EINVAL)?;
+        } else { value.encode(bytes).map_err(|_| EINVAL)?; }
+        objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes)
+    }
+    fn initialize_state(objects: &mut [Buffer], context_slot: u32, profile: super::m3_profile::Profile) -> Result {
+        let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL);
+        let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot };
+        let list_owner = address(JOB_LIST)?;
+        let mut context = [0; state::CONTEXT_SIZE];
+        if profile == super::m3_profile::Profile::Extended {
+            state::context_extended(&mut context).map_err(|_| EINVAL)?;
+        } else { state::context(&mut context).map_err(|_| EINVAL)?; }
+        let context_size = if profile == super::m3_profile::Profile::Extended {
+            state::EXTENDED_CONTEXT_SIZE
+        } else { state::CONTEXT_SIZE };
+        objects[GPU_CONTEXT].write(0, &context[..context_size])?;
+        let mut notifier = KVec::new();
+        notifier.resize(if profile == super::m3_profile::Profile::Extended {
+            state::EXTENDED_NOTIFIER_SIZE
+        } else { state::NOTIFIER_SIZE }, 0u8, GFP_KERNEL)?;
+        if profile == super::m3_profile::Profile::Extended {
+            notification.encode_extended(&mut notifier).map_err(|_| EINVAL)?;
+        } else { notification.encode(&mut notifier).map_err(|_| EINVAL)?; }
+        objects[NOTIFIER].write(0, &notifier)?;
+        let mut list = [0; state::JOB_LIST_SIZE];
+        state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?;
+        objects[JOB_LIST].write(0, &list)?;
+        for (owner, value) in [(storage::COUNTER, 1), (storage::EVENT, 1), (storage::STAMP, timeline::stamp(0xc1000000,0))] {
+            let mut bytes = [0; state::MANAGER_STATE_SIZE];
+            let bytes = bytes.get_mut(..objects[owner].size()).ok_or(EINVAL)?;
+            state::counter(bytes, value).map_err(|_| EINVAL)?;
+            objects[owner].write(0, bytes)?;
+        }
+        // Fill the preemption buffer with 0xcc, including when a compute
+        // command is submitted again.
+        objects[PREEMPTION].fill(0xcc)
+    }
+    pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm,
+        control: crate::m3::agx_compute::Control) -> Result {
+        if !self.complete()? { return Err(EBUSY); }
+        // All prior flushed stamps/events and engine idle were checked by
+        // Runtime before entry. Keep pool/queue ownership, remap only client
+        // views and patch the context when switching Vulkan processes.
+        if !self._binding.matches(vm) {
+            let next = if let Some(index) = self.cached_views.iter().position(|v| v.binding.matches(vm)) {
+                self.cached_views.remove(index).map_err(|_| EIO)?
+            } else {
+                let binding = uat.bind(vm)?;
+                let mut views = KVec::new();
+                for i in 0..storage::COUNT {
+                    if matches!(storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) {
+                        views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?;
+                    }
+                }
+                let command = self.objects[storage::COMMAND].map_gpu_view(vm)?;
+                CachedViews {binding, command, views}
+            };
+            // Complete allocations before moving the active views. Eviction
+            // drops only an idle cached view; no arena storage is duplicated.
+            let old_views = KVec::with_capacity(next.views.len(), GFP_KERNEL)?;
+            self.cached_views.reserve(1, GFP_KERNEL)?;
+            if self.cached_views.len() == 3 {
+                drop(self.cached_views.remove(0).map_err(|_| EIO)?);
+            }
+            let mut old = CachedViews {
+                binding: core::mem::replace(&mut self._binding, next.binding),
+                command: core::mem::replace(&mut self._client_command, next.command),
+                views: old_views,
+            };
+            for (i, view) in next.views {
+                old.views.push((i, self.objects[i].swap_gpu_view(view)?), GFP_KERNEL)?;
+            }
+            self.cached_views.push(old, GFP_KERNEL)?;
+        }
+        self.checkpoint=(self.sequence,self.head,self.batch_count);
+        self.batch_count=0;
+        if let Err(error)=self.append(control) {
+            self.abort_batch()?;
+            return Err(error);
+        }
+        Ok(())
+    }
+    /// Append only while the previous batch is fully retired and this one
+    /// remains unpublished. Firmware serializes each command, including its
+    /// original completion flush, in this one compute queue.
+    pub(crate) fn append(&mut self,control:crate::m3::agx_compute::Control)->Result {
+        if self.batch_count>=storage::SLOTS {return Err(E2BIG);}
+        self.slot=self.batch_count;
+        for i in storage::RESET {
+            let offset=storage::slot_offset_for(self.profile,i,self.slot).map_err(|_|EINVAL)?;
+            let size=storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.size;
+            self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?;
+        }
+        self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?;
+        self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?;
+        Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?;
+        let stamp=self.stamp();
+        Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?;
+        self.write_counters()?;
+        let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64;
+        self.objects[storage::RING].u64(usize::from(self.head)*8,address)?;
+        self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16;
+        self.batch_count+=1;
+        self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?;
+        agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();
+        Ok(())
+    }
+    /// Roll back producer state before any SubmitQueue publication. Old ring
+    /// entries beyond the restored cursor are unreachable and get overwritten.
+    pub(crate) fn abort_batch(&mut self)->Result {
+        (self.sequence,self.head,self.batch_count)=self.checkpoint;
+        self.slot=self.batch_count.saturating_sub(1);
+        self.write_counters()?;
+        self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?;
+        agx_memory::publish();Ok(())
+    }
+    fn write_counters(&mut self) -> Result {
+        for index in [storage::EVENT, storage::COUNTER] {
+            if self.profile == super::m3_profile::Profile::Extended {
+                self.objects[index].u64(0, self.sequence)?;
+            } else { self.objects[index].u32(0, self.sequence as u32)?; }
+        }
+        Ok(())
+    }
+
+    fn offset(&self,owner:usize)->Result<usize> {
+        storage::slot_offset_for(self.profile,owner,self.slot).map_err(|_|EINVAL)
+    }
+    /// Same cache-generation contract as M4: StartCompute and the persistent
+    /// notifier must use the same tag before the firmware flushes user stamps.
+    /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804.
+    pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result {
+        if coalesce && self.profile == super::m3_profile::Profile::Extended { return Err(EINVAL); }
+        let sequence=self.offset(storage::MICROSEQUENCE)?;
+        let command=self.offset(storage::COMMAND)?;
+        self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?;
+        self.objects[storage::COMMAND].u32(command+if self.profile == super::m3_profile::Profile::Extended { 0x81c } else { command::FLUSH_STAMPS },u32::from(!coalesce))?;
+        agx_memory::publish();Ok(())
+    }
+    pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result {
+        let offset=self.offset(storage::COMMAND)?+if self.profile == super::m3_profile::Profile::Extended {
+            0x844
+        } else { command::USER_TIMESTAMPS };
+        self.objects[storage::COMMAND].u64(offset,addresses[0])?;
+        self.objects[storage::COMMAND].u64(offset+8,addresses[1])?;
+        let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64};
+        let sequence=self.offset(storage::MICROSEQUENCE)?;
+        for start in if self.profile == super::m3_profile::Profile::Extended { [0x1bc, 0x20c] }
+            else { [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] } {
+            self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?;
+        }
+        agx_memory::publish();Ok(())
+    }
+    pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result {
+        let sequence=self.offset(storage::MICROSEQUENCE)?;
+        self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended {
+            microsequence::Sequence::EXTENDED_ATTACHMENTS
+        } else { microsequence::ATTACHMENTS },&attachments.0)?;
+        self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended {
+            microsequence::Sequence::EXTENDED_HAS_ATTACHMENTS
+        } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?;
+        agx_memory::publish();Ok(())
+    }
+    fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) }
+    pub(crate) fn completion_dependency(&self)->Result<crate::m3::m3_sync_layout::CompletionDependency> {
+        Ok(crate::m3::m3_sync_layout::CompletionDependency {
+            stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?,
+            value:self.stamp(),event:2,
+        })
+    }
+    pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat)->Result {
+        if self.barriers.is_none() {
+            self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?);
+        }
+        Ok(())
+    }
+    /// Insert a dependency before the last appended, still unpublished CDM.
+    /// Its command ordinal and timestamp slot are unchanged; only the ring
+    /// gains an entry. The whole batch must retire before this storage is reused.
+    pub(crate) fn insert_dependency(&mut self,dependency:crate::m3::m3_sync_layout::CompletionDependency)->Result {
+        if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 {
+            return Err(EINVAL);
+        }
+        let value=self.stamp();
+        let barriers=self.barriers.as_mut().ok_or(EINVAL)?;
+        let offset=self.slot*64;
+        let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE];
+        let barrier=crate::m3::m3_sync_layout::Barrier {
+            stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event,
+            self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass,
+        };
+        if self.profile==super::m3_profile::Profile::Extended {barrier.encode_extended(&mut bytes)}
+        else {barrier.encode(&mut bytes)}.map_err(|_|EINVAL)?;
+        barriers.write(offset,&bytes)?;
+        let address=barriers.va()+offset as u64;
+        let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?;
+        let command=self.objects[storage::COMMAND].va()+self.offset(storage::COMMAND)? as u64;
+        self.objects[storage::RING].u64(usize::from(slots[0])*8,address)?;
+        self.objects[storage::RING].u64(usize::from(slots[1])*8,command)?;
+        self.head=next;
+        self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?;
+        agx_memory::publish();crate::mem::sync();Ok(())
+    }
+    pub(crate) fn gpu_ns(&mut self)->Result<u64> {
+        let mut total: u64 = 0;
+        for slot in 0..self.batch_count {
+            total = total.checked_add(self.slot_gpu_ns(slot)?).ok_or(EOVERFLOW)?;
+        }
+        Ok(total)
+    }
+    pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result<u64> {
+        if slot>=self.batch_count {return Err(EINVAL);}
+        let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?;
+        let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?;
+        if start == 0 || end < start { return Ok(0); }
+        self.profile.gpu_duration_ns(end - start).ok_or(EOVERFLOW)
+    }
+    /// A batch span includes gaps between its compute commands, unlike gpu_ns.
+    /// Read only after stamps, queues, events and idle state prove retirement.
+    pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> {
+        if self.batch_count == 0 {return Err(EINVAL);}
+        Ok([self.profile.profiler_ticks(self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?),
+            self.profile.profiler_ticks(self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?)])
+    }
+    pub(crate) fn head(&self) -> u16 { self.head }
+    pub(crate) fn ordinal(&self)->u64 {self.sequence}
+    pub(crate) fn first(&self) -> bool {
+        self.profile == super::m3_profile::Profile::Legacy && self.sequence == self.batch_count as u64
+    }
+    pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result {
+        // Report real completions by elapsed time as well as sequence count;
+        // batches need not land on a multiple of 128.
+        if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            return Ok(());
+        }
+        let now = <kernel::time::Monotonic as kernel::time::ClockSource>::ktime_get();
+        if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000
+            && self.sequence != 1 && self.sequence % 128 != 0
+            && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            return Ok(());
+        }
+        self.last_progress_ns = now;
+        dev_info!(dev.as_ref(), "M3_COMPUTE_RETIRED sequence={} context={} stamp={:#x}/{:#x} queue={}/{} head={} idle=1 cache_flush=1\n",
+            self.sequence, self._binding.slot(), self.objects[storage::STAMP].read_u32(0)?,
+            self.objects[storage::FW_STAMP].read_u32(0)?, self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?,
+            self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?, self.head);
+        Ok(())
+    }
+
+    pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()}
+    pub(crate) fn complete(&mut self)->Result<bool> {
+        let stamped = self.objects[storage::STAMP].read_u32(0)? == self.stamp()
+            && self.objects[storage::FW_STAMP].read_u32(0)? == self.stamp()
+            && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)? == u32::from(self.head)
+            && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)? == u32::from(self.head);
+        if !stamped { return Ok(false); }
+        if self.profile == super::m3_profile::Profile::Extended {
+            return Ok(self.objects[storage::POOL].read_u64(0x54)? == self.sequence);
+        }
+        Ok(true)
+    }
+    pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {
+        for off in (0..512).step_by(128) {
+            let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?;
+            dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw);
+        }
+        let stamp=self.objects[storage::STAMP].read_u32(0)?;
+        let fw=self.objects[storage::FW_STAMP].read_u32(0)?;
+        let done=self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?;
+        let read=self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?;
+        dev_info!(dev.as_ref(),"M3 compute: stamps={:#x}/{:#x} done={} read={}\n",stamp,fw,done,read);
+        let mut record=[0u8;0x84];self.objects[storage::POOL].read(0,&mut record)?;
+        dev_info!(dev.as_ref(),"M3 compute pool={:02x?}\n",record);
+        let mut roots=[(0,0);64];self._binding.vm().context_roots(&mut roots)?;
+        dev_info!(dev.as_ref(),"M3 context {} roots={:x?}, bootstrap={:x?}\n",self._binding.slot(),roots[self._binding.slot() as usize],roots[0]);
+        Ok(())
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs
index 74bed858b71e2..ace58e38f38a6 100644
--- a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 use super::{m3_pool_layout::GpuRegion, m3_queue_layout::{Error, FirmwareVa}};
 pub(crate) const SIZE: usize = 0x893;
+pub(crate) const EXTENDED_SIZE: usize = 0x8b0;
 pub(crate) const REGISTERS: usize = 0x20;
 pub(crate) const REGISTER_COUNT: usize = 22;
 /// G15G takes the same list without its last two registers (0xa599, 0xd411).
@@ -64,12 +65,29 @@ impl Command {
     /// Validate before any output mutation. Encoding operates on ordinary host
     /// bytes; the caller copies them into the owned unpublished command buffer.
     pub(crate) fn encode(&self, out: &mut [u8]) -> Result<(), Error> {
-        if out.len() < SIZE { return Err(Error::Size); }
+        self.encode_for(super::m3_profile::Profile::Legacy, 0, 0, out)
+    }
+
+    /// Firmware envelopes vary independently of the GPU's CDM stream format.
+    pub(crate) fn encode_for(&self, profile: super::m3_profile::Profile,
+        asid_generation: u8, event_slot: u8, out: &mut [u8]) -> Result<(), Error> {
+        self.encode_with_usc(profile, asid_generation, event_slot, 0, out)
+    }
+
+    pub(crate) fn encode_with_usc(&self, profile: super::m3_profile::Profile,
+        asid_generation: u8, event_slot: u8, usc_base: u64, out: &mut [u8]) -> Result<(), Error> {
+        let extended = profile == super::m3_profile::Profile::Extended;
+        if usc_base & 0xffff_ffff != 0 || usc_base >= 1 << 42
+            || (!extended && usc_base != 0) { return Err(Error::Address); }
+        let size = if extended { EXTENDED_SIZE } else { SIZE };
+        if out.len() < size { return Err(Error::Size); }
+        if extended && (self.register_set != RegisterSet::G15G || event_slot >= 4
+            || (self.context != 0 && asid_generation == 0)) { return Err(Error::Address); }
         if self.context >= 64 || self.cdm_last.get() < self.cdm.get() { return Err(Error::Address); }
         if self.preemption.size() < 0x14a0 || self.microsequence_size == 0 || self.microsequence_size > 4096 {
             return Err(Error::Size);
         }
-        self.gpu_alias.add((SIZE - 3) as u64)?; // inclusive final aligned dword
+        self.gpu_alias.add(((size - 1) & !3) as u64)?; // inclusive final aligned dword
         let registers = self.gpu_alias.add(REGISTERS as u64)?;
         let preemption = self.preemption.base();
         let values = [
@@ -82,45 +100,157 @@ impl Command {
             (0x1a0b9, 0), (0x1a0c1, 0), (0x101d1, 0), (0xd479, 0),
             (0x1a0e9, 8), (0x107a1, 0xff0000), (0xa599, 0x13200400020), (0xd411, 0x200000001),
         ];
-        let bytes = &mut out[..SIZE];bytes.fill(0);
+        let bytes = &mut out[..size];bytes.fill(0);
         word(bytes, 0, 3); // compute opcode
-        pointer(bytes, COUNTER, u64::from(self.counter));
-        word(bytes, CONTEXT, self.context);
-        pointer(bytes, 0x14, self.notifier.get());
-        let count = match self.register_set {
+        if extended {
+            word(bytes, 0xc, self.context);
+            pointer(bytes, 0x10, self.notifier.get());
+        } else {
+            pointer(bytes, COUNTER, u64::from(self.counter));
+            word(bytes, CONTEXT, self.context);
+            pointer(bytes, 0x14, self.notifier.get());
+        }
+        let legacy_count = match self.register_set {
             RegisterSet::G15S => REGISTER_COUNT,
             RegisterSet::G15G => G15G_REGISTER_COUNT,
         };
-        for (n, (register, value)) in values.into_iter().take(count).enumerate() {
-            word(bytes, REGISTERS+n*REGISTER_STRIDE, register);
-            pointer(bytes, REGISTERS+n*REGISTER_STRIDE+4, value);
+        let mut count = 0;
+        let mut emit = |register, value| {
+            word(bytes, REGISTERS + count * REGISTER_STRIDE, register);
+            pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value);
+            count += 1;
+        };
+        let limit = if extended { REGISTER_COUNT } else { legacy_count };
+        for (register, value) in values.into_iter().take(limit) {
+            if extended && register == 0x12091 { continue; }
+            emit(register, value);
+        }
+        if extended {
+            emit(0x1a540, 1);
+            emit(0x014a9, 1);
+            emit(0x0a351, 1);
+            if usc_base != 0 { emit(0x10071, usc_base); }
         }
         pointer(bytes, 0x720, registers.get());
         bytes[0x728..0x72a].copy_from_slice(&(count as u16).to_le_bytes());
         bytes[0x72a..0x72c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes());
         pointer(bytes, 0x760, self.microsequence.get());
         word(bytes, 0x768, self.microsequence_size);
-        // ComputeInfo2 mirrors the preemption buffer, inclusive CDM end and
-        // qualified control word from the register list.
-        pointer(bytes, 0x76c+0x28, preemption);
-        pointer(bytes, 0x76c+0x30, self.cdm_last.get());
-        pointer(bytes, 0x76c+0x58, QUALIFIED_COMPUTE_CONTROL);
-        word(bytes, 0x7cc+0x14, u32::MAX); // EncoderParams.unk_mask
-        pointer(bytes, META, self.stamp.get());
-        pointer(bytes, META+8, self.fw_stamp.get());
-        word(bytes, STAMP_VALUE, self.stamp_value);
-        word(bytes, META+0x14, 2); // qualified compute event slot
-        word(bytes, FLUSH_STAMPS, u32::from(self.flush_stamps));
-        word(bytes, META+0x20, 0xc10000); // producer UUID
-        word(bytes, QUEUE_COUNT, self.counter);
-        pointer(bytes, TAIL+8, self.timestamps[0].get());
-        pointer(bytes, TAIL+16, self.timestamps[1].get());
-        pointer(bytes, TAIL+0x2e, self.pool.get());
-        bytes[TAIL+0x36] = 1; // SharedCommandTail.unk_846
-        pointer(bytes, TAIL+0x37, 0xad0000);
-        pointer(bytes, TAIL+0x3f, 0x1600000);
-        pointer(bytes, TAIL+0x47, self.tail_scratch.get());
-        bytes[TAIL+0x4f] = 1; // SharedCommandTail.unk_85f
+        let info = if extended { 0x770 } else { 0x76c };
+        pointer(bytes, info + 0x28, preemption);
+        pointer(bytes, info + 0x30, self.cdm_last.get());
+        pointer(bytes, info + 0x58, QUALIFIED_COMPUTE_CONTROL);
+        word(bytes, if extended { 0x7e8 } else { 0x7e0 }, u32::MAX);
+        let meta = if extended { 0x800 } else { META };
+        pointer(bytes, meta, self.stamp.get());
+        pointer(bytes, meta + 8, self.fw_stamp.get());
+        word(bytes, meta + 0x10, self.stamp_value);
+        word(bytes, meta + 0x14, 2);
+        if extended { word(bytes, meta + 0x18, event_slot.into()); }
+        word(bytes, meta + 0x1c, u32::from(self.flush_stamps));
+        word(bytes, meta + 0x20, 0xc10000);
+        word(bytes, meta + 0x24, self.counter);
+        let tail = if extended { 0x82c } else { TAIL };
+        pointer(bytes, tail + 8, self.timestamps[0].get());
+        pointer(bytes, tail + 16, self.timestamps[1].get());
+        pointer(bytes, tail + if extended { 0x3e } else { 0x2e }, self.pool.get());
+        bytes[tail + if extended { 0x46 } else { 0x36 }] = 1;
+        pointer(bytes, tail + if extended { 0x47 } else { 0x37 },
+            if extended { 0x1588000 } else { 0xad0000 });
+        pointer(bytes, tail + if extended { 0x4f } else { 0x3f },
+            if extended { 0xef8000 } else { 0x1600000 });
+        pointer(bytes, tail + if extended { 0x57 } else { 0x47 }, self.tail_scratch.get());
+        if extended { bytes[0x88b] = asid_generation; }
+        else { bytes[TAIL + 0x4f] = 1; }
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use super::super::m3_profile::Profile;
+
+    fn fixture() -> Command {
+        let firmware = |n: u64| FirmwareVa::new(0xffff_fc21_8000_0000 + n * 0x4000).unwrap();
+        Command {
+            context: 3, counter: 7, notifier: firmware(0),
+            preemption: GpuRegion::new(0x5000_0000, 0x4000).unwrap(),
+            cdm: GpuVa::new(0x6000_0000).unwrap(),
+            cdm_last: GpuVa::new(0x6000_0ffc).unwrap(),
+            gpu_alias: GpuVa::new(0x7000_0000).unwrap(),
+            microsequence: firmware(1), microsequence_size: 0x300,
+            stamp: firmware(2), fw_stamp: firmware(3), stamp_value: 0x700,
+            timestamps: [firmware(4), firmware(5)], pool: firmware(6),
+            tail_scratch: firmware(7), flush_stamps: true,
+            register_set: RegisterSet::G15G,
+        }
+    }
+    fn read32(b: &[u8], o: usize) -> u32 {
+        u32::from_le_bytes(b[o..o + 4].try_into().unwrap())
+    }
+    fn read64(b: &[u8], o: usize) -> u64 {
+        u64::from_le_bytes(b[o..o + 8].try_into().unwrap())
+    }
+
+    #[test]
+    fn extended_envelope_preserves_stream_and_native_visibility_boundary() {
+        let c = fixture();
+        let mut b = [0xa5; EXTENDED_SIZE + 16];
+        c.encode_for(Profile::Extended, 9, 2, &mut b).unwrap();
+        assert_eq!(read32(&b, 0xc), c.context);
+        assert_eq!(read64(&b, 0x10), c.notifier.get());
+        assert_eq!(read32(&b, 0x728) & 0xffff, 24);
+        let count = (read32(&b, 0x728) & 0xffff) as usize;
+        let cdm = (0..count).find(|n| read32(&b, 0x20 + n * 12) == 0x1a420).unwrap();
+        assert_eq!(read64(&b, 0x24 + cdm * 12), c.cdm.get());
+        assert_eq!(read64(&b, 0x7a0), c.cdm_last.get());
+        assert_eq!(read64(&b, 0x800), c.stamp.get());
+        assert_eq!(read64(&b, 0x808), c.fw_stamp.get());
+        assert_eq!(read32(&b, 0x818), 2);
+        assert_eq!(read32(&b, 0x81c), 1);
+        assert_eq!(read64(&b, 0x834), c.timestamps[0].get());
+        assert_eq!(read64(&b, 0x86a), c.pool.get());
+        assert_eq!(read64(&b, 0x883), c.tail_scratch.get());
+        assert_eq!(b[0x88b], 9);
+        assert_eq!(&b[EXTENDED_SIZE..], &[0xa5; 16]);
+    }
+
+    #[test]
+    fn extended_validation_never_mutates_a_rejected_envelope() {
+        for (generation, slot) in [(0, 0), (1, 4), (1, 255)] {
+            let mut b = [0xa5; EXTENDED_SIZE];
+            assert!(fixture().encode_for(Profile::Extended, generation, slot, &mut b).is_err());
+            assert_eq!(b, [0xa5; EXTENDED_SIZE]);
+        }
+        let mut b = [0xa5; EXTENDED_SIZE - 1];
+        assert!(fixture().encode_for(Profile::Extended, 1, 0, &mut b).is_err());
+        assert_eq!(b, [0xa5; EXTENDED_SIZE - 1]);
+    }
+
+    #[test]
+    fn legacy_wrapper_keeps_legacy_geometry_and_register_order() {
+        let c = fixture();
+        let mut b = [0xa5; EXTENDED_SIZE];
+        c.encode(&mut b).unwrap();
+        assert_eq!(read32(&b, CONTEXT), c.context);
+        assert_eq!(read64(&b, 0x14), c.notifier.get());
+        assert_eq!(read32(&b, 0x728) & 0xffff, 20);
+        assert_eq!(read32(&b, REGISTERS + 8 * REGISTER_STRIDE), 0x12091);
+        assert_eq!(read64(&b, META), c.stamp.get());
+        assert_eq!(&b[SIZE..], &[0xa5; EXTENDED_SIZE - SIZE]);
+    }
+    #[test]
+    fn optional_usc_window_is_a_dispatch_register_without_stream_rewriting() {
+        let c = fixture();
+        let mut output = [0xa5; EXTENDED_SIZE];
+        c.encode_with_usc(Profile::Extended, 1, 0, 1 << 32, &mut output).unwrap();
+        assert_eq!(read32(&output, 0x728) & 0xffff, 25);
+        assert_eq!(read32(&output, REGISTERS + 24 * REGISTER_STRIDE), 0x10071);
+        assert_eq!(read64(&output, REGISTERS + 24 * REGISTER_STRIDE + 4), 1 << 32);
+        output.fill(0xa5);
+        assert!(c.encode_with_usc(Profile::Extended, 1, 0, 1, &mut output).is_err());
+        assert_eq!(output, [0xa5; EXTENDED_SIZE]);
+    }
+
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs
index a10c3040d570f..bd78c03823aa0 100644
--- a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs
@@ -82,3 +82,113 @@ impl Sequence {
         Ok(())
     }
 }
+
+impl Sequence {
+    pub(crate) const EXTENDED_LENGTH: usize = 0x300;
+    pub(crate) const EXTENDED_ATTACHMENTS: usize = 0x58;
+    pub(crate) const EXTENDED_HAS_ATTACHMENTS: usize = 0x2c0;
+
+    /// The extended firmware operates on a native G15 CDM stream. Its firmware
+    /// microsequence has wider timestamp operations and different engine state.
+    pub(crate) fn encode_extended(&self, auxiliary: command::GpuVa,
+        context_store: super::m3_pool_layout::GpuRegion, queue_sequence: u64,
+        output: &mut [u8]) -> Result<(), Error> {
+        if self.context >= 64 || self.generation != 0 { return Err(Error::Address); }
+        if context_store.size() < 0x20000 { return Err(Error::Size); }
+        let registers = self.command.at(command::REGISTERS, 24 * command::REGISTER_STRIDE)?;
+        let engine = self.command.at(0x76c, 0x60)?;
+        let clock = self.command.at(0x82c, 8)?;
+        let pairs = self.command.at(0x834, 16)?;
+        let end_pair = self.command.at(0x83c, 8)?;
+        let pool_pointer = self.command.at(0x86a, 8)?;
+        let status = self.command.at(0x8ac, 4)?;
+        let time_aux = self.command.at(0x89c, 8)?;
+        let elapsed = self.command.at(0x7dc, 8)?;
+        let tail = self.command.at(0x88c, 8)?;
+        let activity = self.stats.at(0, 8)?;
+        let bytes = output.get_mut(..STORAGE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        word(bytes, 0, 0x0b);
+        pointer(bytes, 0x14, registers);
+        pointer(bytes, 0x1c, activity);
+        pointer(bytes, 0x24, self.queue.get());
+        word(bytes, 0x2c, self.context);
+        word(bytes, 0x30, 1);
+        pointer(bytes, 0x38, queue_sequence);
+        pointer(bytes, 0x44, engine);
+        word(bytes, 0x50, 1);
+        pointer(bytes, 0x160, self.pool.get());
+        pointer(bytes, 0x178, auxiliary.get());
+        for (offset, delta) in [0, 0xf400, 0x1e800, 0x1f000].into_iter().enumerate() {
+            pointer(bytes, 0x188 + offset * 8, context_store.base() + delta);
+        }
+        pointer(bytes, 0x1a8, status);
+        word(bytes, 0x1b8, 2);
+        // Timestamp records are 76 bytes; WFI sits between the two records.
+        for (offset, opcode, selected) in [(0x1bc, 0x8000_0003, pairs), (0x20c, 3, end_pair)] {
+            word(bytes, offset, opcode);
+            pointer(bytes, offset + 4, clock);
+            pointer(bytes, offset + 0xc, pairs);
+            pointer(bytes, offset + 0x14, selected);
+            pointer(bytes, offset + 0x1c, self.queue.get());
+            pointer(bytes, offset + 0x2c, time_aux);
+            pointer(bytes, offset + 0x34, elapsed);
+            pointer(bytes, offset + 0x44, 0xc10000);
+        }
+        word(bytes, 0x208, 1);
+        let finish = 0x258;
+        word(bytes, finish, 0x0c);
+        pointer(bytes, finish + 4, activity);
+        pointer(bytes, finish + 0xc, self.queue.get());
+        word(bytes, finish + 0x14, self.context);
+        pointer(bytes, finish + 0x18, engine);
+        pointer(bytes, finish + 0x24, 0xc10000);
+        pointer(bytes, finish + 0x2c, self.fw_stamp.get());
+        word(bytes, finish + 0x34, self.stamp_value);
+        pointer(bytes, finish + 0x5c, pool_pointer);
+        word(bytes, finish + 0x64, (-(finish as i32)) as u32);
+        pointer(bytes, finish + 0x69, status);
+        pointer(bytes, finish + 0x71, tail);
+        word(bytes, 0x2d4, 0x4000_0002);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    fn fixture() -> Sequence {
+        let fw = |i: u64| FirmwareVa::new(0xffff_fc21_a000_0000 + i * 0x4000).unwrap();
+        Sequence { command: Region::new(fw(0).get(), command::EXTENDED_SIZE).unwrap(),
+            stats: Region::new(fw(1).get(), 0xe10).unwrap(),
+            notifier: Region::new(fw(2).get(), 0x200).unwrap(), queue: fw(3),
+            context: 3, generation: 0, pool: fw(4), scratch: [fw(5); 5],
+            fw_stamp: fw(6), stamp_value: 0x700 }
+    }
+    fn ptr(bytes: &[u8], at: usize) -> u64 { u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) }
+    #[test] fn extended_timestamps_and_retirement_refer_to_their_live_owners() {
+        let s = fixture();
+        let mut output = [0xa5; STORAGE + 8];
+        s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(),
+            super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 7, &mut output).unwrap();
+        assert_eq!(ptr(&output, 0x1c), s.stats.at(0, 8).unwrap());
+        assert_eq!(ptr(&output, 0x38), 7);
+        assert_eq!(ptr(&output, 0x188), 0x6000_0000);
+        assert_eq!(ptr(&output, 0x1bc + 0x14), s.command.at(0x834, 8).unwrap());
+        assert_eq!(ptr(&output, 0x20c + 0x14), s.command.at(0x83c, 8).unwrap());
+        assert_eq!(ptr(&output, 0x258 + 0x2c), s.fw_stamp.get());
+        assert_eq!(ptr(&output, 0x258 + 0x5c), s.command.at(0x86a, 8).unwrap());
+        assert_eq!(&output[STORAGE..], &[0xa5; 8]);
+    }
+    #[test] fn extended_sequence_rejects_legacy_command_extents_and_event_generations() {
+        for generation in [0, 1] {
+            let mut s = fixture();
+            s.generation = generation;
+            if generation == 0 { s.command = Region::new(s.command.at(0, 1).unwrap(), command::SIZE).unwrap(); }
+            let mut output = [0xa5; STORAGE];
+            assert!(s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(),
+                super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 0, &mut output).is_err());
+            assert_eq!(output, [0xa5; STORAGE]);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs
index 6bd3aecda387d..a897a45f0922d 100644
--- a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs
@@ -87,3 +87,69 @@ pub(crate) fn allocation(index:usize)->Result<Allocation,InvalidOwner> {
 pub(crate) fn coalesce_stamp_flush(wide:bool,timestamps:[u64;2],last:bool)->bool {
     wide && timestamps==[0,0] && !last
 }
+
+pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize)
+    -> Result<Allocation, InvalidOwner> {
+    let mut owner = allocation(index)?;
+    if profile == super::m3_profile::Profile::Legacy { return Ok(owner); }
+    match index {
+        // The initial allocator has eight 4-MiB payload owners. Retain the
+        // historical indices without allocating unused 4-MiB payloads.
+        9..=22 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; }
+        POOL_PAGES => owner.size = 0x20000,
+        POOL_TABLE => { owner.size = 0x4000; owner.mapping = Mapping::Client(POOL_TABLE_VA & !0x3fff); }
+        POOL => owner.size = 0xa0,
+        GPU_CONTEXT => owner.size = 0x38,
+        NOTIFIER => owner.size = 0x200,
+        EVENT => owner.size = 8,
+        QUEUE => owner.size = 0x24c0,
+        PREEMPTION => owner.size = 0x4000,
+        COMMAND => { owner.size = 0x8b0; owner.mapping = Mapping::CommandAlias(0x73_0000_0000); }
+        TAIL_SCRATCH => { owner.size = 0x4000; owner.mapping = Mapping::Client(0x72_0100_0000); }
+        46 => { owner.size = 0x40000; owner.mapping = Mapping::Client(0x72_0000_0000); }
+        45 | 47..=49 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; }
+        _ => {}
+    }
+    Ok(owner)
+}
+
+pub(crate) fn slot_offset_for(profile: super::m3_profile::Profile, index: usize, slot: usize)
+    -> Result<usize, InvalidOwner> {
+    if slot >= SLOTS { return Err(InvalidOwner); }
+    let size = allocation_for(profile, index)?.size;
+    Ok(if RESET.contains(&index) { ((size + 0x3fff) & !0x3fff) * slot } else { 0 })
+}
+
+pub(crate) fn batch_allocation_for(profile: super::m3_profile::Profile, index: usize)
+    -> Result<Allocation, InvalidOwner> {
+    let mut owner = allocation_for(profile, index)?;
+    owner.size += slot_offset_for(profile, index, SLOTS - 1)?;
+    Ok(owner)
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    use super::super::m3_profile::Profile;
+    #[test] fn extended_commands_do_not_straddle_another_slots_page() {
+        let command = allocation_for(Profile::Extended, COMMAND).unwrap();
+        let Mapping::CommandAlias(base) = command.mapping else { panic!("missing command alias") };
+        assert_eq!(base & 0x3fff, 0);
+        for slot in 0..SLOTS - 1 {
+            let start = slot_offset_for(Profile::Extended, COMMAND, slot).unwrap();
+            let next = slot_offset_for(Profile::Extended, COMMAND, slot + 1).unwrap();
+            assert!(start + command.size <= next);
+        }
+        assert_eq!(batch_allocation_for(Profile::Extended, QUEUE).unwrap().size, 0x24c0);
+        assert!(slot_offset_for(Profile::Extended, COMMAND, SLOTS).is_err());
+    }
+    #[test] fn legacy_plan_is_byte_for_byte_the_same_for_every_owner_and_slot() {
+        for index in 0..COUNT {
+            assert_eq!(allocation_for(Profile::Legacy, index), allocation(index));
+            for slot in 0..SLOTS {
+                assert_eq!(slot_offset_for(Profile::Legacy, index, slot), slot_offset(index, slot));
+            }
+            assert_eq!(batch_allocation_for(Profile::Legacy, index), batch_allocation(index));
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs
new file mode 100644
index 0000000000000..02b847e210dc9
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_config.rs
@@ -0,0 +1,661 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Owned RTKit 2419.140.12 configuration with named initialization references.
+use kernel::prelude::*;
+use crate::m3::m3_init_layout as init;
+use crate::m3::m3_init_storage as storage;
+use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD,
+    UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS,
+    FW_CONTROL_STATE,FW_CONTROL_RING};
+use crate::{driver,m3::agx_memory::{self},m3::m3_memory::Buffer,mmu,pgtable::prot};
+kernel::static_assert!(core::mem::size_of::<crate::fw::channels::RawFwLogMsg>() == init::fwlog::ENTRY_SIZE);
+kernel::static_assert!(core::mem::size_of::<crate::fw::channels::RawFwLogPayloadMsg>() == init::fwlog::PAYLOAD_SIZE);
+kernel::static_assert!(<crate::fw::channels::FwLogChannelState as crate::fw::channels::RxChannelState>::SUB_CHANNELS == init::fwlog::CHANNELS);
+
+/// The root, runtime-pointer and firmware-control records, encoded for the owners
+/// `region` describes. Config uploads them for its own allocations; the device-tree
+/// InitData checks encode them for the fixed allocation table before GPU access.
+pub(crate) struct Records {
+    pub(crate) root: [u8; init::ROOT_SIZE],
+    pub(crate) runtime: [u8; init::RUNTIME_SIZE],
+    pub(crate) control: KVec<u8>,
+}
+impl Records {
+    pub(crate) fn encode(region: impl Fn(usize) -> Result<init::Region>) -> Result<Self> {
+        let root = init::Root { region_a: region(REGION_A)?, runtime: region(RUNTIME_POINTERS)?,
+            globals: region(GLOBALS)?, new_a: region(RUNTIME_FLAGS)?,
+            control: region(CONTROL_REGION)?, power: region(GLOBALS_POWER)? };
+        let mut channels = [init::Channel { state: region(0)?, ring: region(1)? }; 17];
+        for (index, channel) in channels.iter_mut().enumerate() {
+            *channel = init::Channel { state: region(index * 2)?, ring: region(index * 2 + 1)? };
+        }
+        let runtime = init::RuntimePointers { hardware: region(HARDWARE_DATA)?,
+            unknown_pair: region(UNKNOWN_PAIR)?, fwlog_payload: region(FWLOG_PAYLOAD)?,
+            unknown_c0: region(UNKNOWN_C0)?, unknown_c1: region(UNKNOWN_C1)?,
+            unknown_c3: region(UNKNOWN_C3)?, channels };
+        let control = init::ControlRegion { channel: init::Channel {
+            state: region(FW_CONTROL_STATE)?, ring: region(FW_CONTROL_RING)? } };
+        let mut root_bytes = [0; init::ROOT_SIZE];
+        root.encode(&mut root_bytes).map_err(|_| EINVAL)?;
+        let mut runtime_bytes = [0; init::RUNTIME_SIZE];
+        runtime.encode(&mut runtime_bytes).map_err(|_| EINVAL)?;
+        // The control/status region is 50 KiB; use temporary heap storage,
+        // never a kernel-stack array. All three owners precede firmware startup.
+        let mut control_bytes = KVec::new();
+        control_bytes.resize(init::CONTROL_SIZE, 0u8, GFP_KERNEL)?;
+        control.encode(&mut control_bytes).map_err(|_| EINVAL)?;
+        Ok(Self { root: root_bytes, runtime: runtime_bytes, control: control_bytes })
+    }
+}
+
+pub(crate) struct Config {
+    objects: KVec<Buffer>,
+    profile: super::m3_profile::Profile,
+    control_generation: u32,
+    io_descriptors: Option<KVec<u8>>,
+    _iomaps: KVec<mmu::KernelMapping>,
+    pub(crate) completed_events: u64,
+    pstates: crate::m3::m3_adt_config::PstateWatch,
+    thermal: crate::m3::m3_thermal::Governor,
+    request_top: u32,
+    request_value: u32,
+    request_checked: bool,
+    request_check_at: Option<kernel::time::Instant<kernel::time::Monotonic>>,
+    request_writes: u64,
+    request_observed: u32,
+}
+impl Config {
+
+    pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware,
+        contents: &crate::m3::m3_adt_config::Contents) -> Result<Self> {
+        contents.check_for_upload()?;
+        if let Some(inputs) = &contents.extended {
+            if firmware.profile() != super::m3_profile::Profile::Extended { return Err(EINVAL); }
+            let owners = ExtendedStorage::new(dev, uat, &inputs.power, &inputs.sram_uv, inputs.bases)?;
+            let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?;
+            return Ok(Self { objects: owners.objects, _iomaps: owners.iomaps,
+                profile: firmware.profile(), control_generation: 0, completed_events: 0,
+                io_descriptors: Some(owners.descriptors),
+                pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal,
+                request_top: 8, request_value: 0, request_checked: false, request_check_at: None,
+                request_writes: 0, request_observed: 0 });
+        }
+        let mut objects = KVec::new();
+        let mut iomaps = KVec::new();
+        for index in 0..storage::COUNT {
+            let mut a=storage::allocation(index).map_err(|_|EINVAL)?;
+            let image=contents.initial(index,&a)?;
+            if a.initial!=storage::Initial::Zero && image.len()!=a.size {return Err(EINVAL);}
+            // The SoC may place the HwData object elsewhere and have it mapped cacheable
+            // (`Soc::hwdata_object`); the records and the upload check use the live address.
+            let object=if index==HARDWARE_DATA {contents.hwdata_object} else {None};
+            if let Some(o)=object {a.address=o.address;}
+            let mut b=match object {
+                Some(o) if o.cached=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address),a.size,
+                    prot::PROT_FW_PRIV_RW,prot::PROT_GPU_SHARED_RW)?,
+                _=>Buffer::at(dev,uat.kernel_vm(),None,Some(a.address),a.size,a.gpu_shared)?,
+            };
+            b.write(0,image)?;objects.push(b,GFP_KERNEL)?;
+        }
+        for io in contents.iomaps {
+            // Read-write unless the contents say otherwise (only the T8122 start experiment).
+            let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW}
+                else {crate::m3::t8122_start::mmio_prot(contents.read_only_slots,io.slot)};
+            let map=uat.kernel_vm().map_io_m3(io.address,io.physical.try_into()?,io.size,protection)?;
+            let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?;
+            let field=io.pointer_field().map_err(|_|EINVAL)?;
+            objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?;
+            iomaps.push(map,GFP_KERNEL)?;
+        }
+        // This field is a physical address owned by the reserved GPU region,
+        // unlike the firmware virtual references in the IOMapping records.
+        objects[HARDWARE_DATA].u64(storage::GPU_REGION_PHYSICAL,firmware.resources.regions[0].base)?;
+        Self::initialize_records(&mut objects)?;
+        // The root record carries the G15 14.8.3 InitData version, which is every T6030
+        // image's. Only the T8122 start experiment gives the firmware another one
+        // (`asahi.t8122_initdata_version`).
+        if firmware.initdata_magic!=crate::m3::m3_firmware::G15_V14_8_3_INITDATA {
+            objects[storage::ROOT].u64(0,firmware.initdata_magic)?;
+        }
+        // Vertex/tessellation loops need not retire a primitive at every
+        // firmware progress poll. The inherited interval of 10 falsely
+        // declares a valid 35.7 ms TA job stuck on J514S. Keep progress
+        // detection enabled, with a finite interval of 100 firmware polls.
+        // This is independent of the host job bound and 60-second watchdog.
+        // RTKit 2419 copies Globals+0x980 into its TA poll threshold at
+        // text+0x6514; text+0xd710 checks it before sampling engine progress.
+        const TA_PROGRESS_INTERVAL: usize = 0x980;
+        if objects[GLOBALS].read_u32(TA_PROGRESS_INTERVAL)? != 10 { return Err(EINVAL); }
+        objects[GLOBALS].u32(TA_PROGRESS_INTERVAL, 100)?;
+        dev_info!(dev.as_ref(), "M3: firmware TA progress-check interval=100\n");
+        // RTKit 2419 +2de48..2dec8 accepts user timestamp stores only
+        // within the 64 MiB arena at HwDataB+28. Match map_timestamp().
+        objects[HARDWARE_DATA].u64(0x28,agx_memory::TIMESTAMP_RANGE.start)?;
+        // RTKit 2419 HwDataA follows the 0x4580-byte HwDataB. Request the
+        // boot performance state through firmware (asahi.m3_boot_pstate) and cap
+        // the states it may use (asahi.m3_max_pstate), retaining the voltage
+        // tables, power/thermal ceilings and minimum operating state.
+        // The bring-up image initialized actual/target and every base to 1
+        // (338 MHz), so short desktop jobs never left that startup state.
+        // The uploaded table is the SoC's (`Soc::pstates`): its highest state at +0xb58 and that
+        // state's frequency in the table at +0xb5c (8 and 1380 MHz on T6030).
+        let (states, top_mhz) = contents.table;
+        if objects[HARDWARE_DATA].read_u32(0xb58)? != states
+            || objects[HARDWARE_DATA].read_u32(0xb5c + states as usize * 4)? != top_mhz
+        {
+            return Err(EINVAL);
+        }
+        let (hwdata, globals) = objects.split_at_mut(GLOBALS);
+        crate::m3::m3_adt_config::publish_pstates(dev.as_ref().as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?;
+        // Firmware ktrace is for explicit detailed diagnostics. Normal games
+        // still drain error/events, but need not generate trace records for
+        // every firmware operation on the command-processing core.
+        objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled(
+            crate::debug::DebugFlags::SubmitTiming)))?;
+        crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?;
+        agx_memory::publish();
+        let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?;
+        Ok(Self { objects, profile: firmware.profile(), control_generation: 0, io_descriptors: None,
+            _iomaps: iomaps, completed_events: 0,
+            pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal,
+            request_top:contents.table.0,request_value:0,request_checked:false,
+            request_check_at:None,request_writes:0,request_observed:0 })
+    }
+    fn initialize_records(objects: &mut [Buffer]) -> Result {
+        let records = Records::encode(|index: usize| {
+            init::Region::new(objects[index].va(), objects[index].size()).map_err(|_| EINVAL)
+        })?;
+        objects[storage::ROOT].write(0, &records.root)?;
+        objects[RUNTIME_POINTERS].write(0, &records.runtime)?;
+        objects[CONTROL_REGION].write(0, &records.control)
+    }
+    pub(crate) fn publish_io_table(&self,
+        pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+        firmware: &crate::m3::m3_firmware::Firmware) -> Result {
+        if let Some(descriptors) = &self.io_descriptors {
+            firmware.publish_io_table(pdev, descriptors)?;
+        }
+        Ok(())
+    }
+
+    pub(crate) fn global_table_views(&mut self, vm: &mmu::Vm) -> Result<KVec<mmu::KernelMapping>> {
+        let mut mappings = KVec::new();
+        if self.profile == super::m3_profile::Profile::Extended {
+            for index in [super::m3_init_26::GLOBAL_POOL, super::m3_init_26::GLOBAL_PARAMETERS] {
+                mappings.push(self.objects[index].map_gpu_view(vm)?, GFP_KERNEL)?;
+            }
+        }
+        Ok(mappings)
+    }
+
+    pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() }
+    pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result {
+        let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?;
+        dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n",
+            u8::from(crate::m3::m3_params::globals_debug_off()),value,crate::m3::m3_params::consistent_perf());
+        Ok(())
+    }
+    pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result {
+        // Native controller inputs supersede the legacy fixed-request interface.
+        if self.profile == super::m3_profile::Profile::Extended { return Ok(()); }
+        let Some(value)=crate::m3::m3_consistent_perf::request(chip,
+            crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());};
+        if value==0 && self.request_value==0 {return Ok(());}
+        let now=kernel::time::Instant::<kernel::time::Monotonic>::now();
+        let elapsed=self.request_check_at.map(|last|(now-last).as_nanos());
+        if !crate::m3::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) {
+            return Ok(());
+        }
+        type P=crate::m3::g15_raw::raw::PowerCtlBlock;
+        let at=core::mem::offset_of!(P,consistent_perf_state);
+        if !self.request_checked {
+            let mut flag=[0];
+            self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut flag)?;
+            if flag[0]!=0 || self.objects[GLOBALS_POWER].read_u32(at)?!=0 {return Err(EINVAL);}
+            self.request_checked=true;
+        }
+        let observed=self.objects[GLOBALS_POWER].read_u32(at)?;
+        self.request_observed=observed;
+        if observed!=value {
+            self.objects[GLOBALS_POWER].packed_request(at,value)?;
+            agx_memory::publish();
+            self.request_writes=self.request_writes.saturating_add(1);
+        }
+        self.request_value=value;
+        self.request_check_at=Some(now);
+        Ok(())
+    }
+    /// Move the runtime cap by temperature (the thermal limit), then check the performance
+    /// state the firmware reports against the ceiling and the runtime cap. Called after boot and
+    /// after every job, by the job thread.
+    pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended {
+            let state = device.pstate_register()? & 15;
+            if state > self.request_top {
+                dev_err!(dev.as_ref(), "M3: native performance state {} exceeds {} ({})\n", state, self.request_top, what);
+                return Err(ERANGE);
+            }
+            return Ok(());
+        }
+        let now = kernel::time::Instant::<kernel::time::Monotonic>::now();
+        if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) {
+            crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?;
+        }
+        let allowed = self.thermal.allowed(now);
+        let (over, peak) = self.pstates.check(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?;
+        self.thermal.note_check(dev.as_ref().as_ref(), now, over, peak)?;
+        self.thermal.busy(now);
+        Ok(())
+    }
+    pub(crate) fn render_stats(&self) -> Result<[init::Region;2]> {
+        if self.profile==super::m3_profile::Profile::Legacy {
+            let value=self.stats_region()?;return Ok([value;2]);
+        }
+        let region=|index:usize,offset:usize| {
+            let object=&self.objects[index];
+            init::Region::new(object.va().checked_add(offset as u64).ok_or(EOVERFLOW)?,
+                object.size().checked_sub(offset).ok_or(EINVAL)?).map_err(|_|EINVAL)
+        };
+        Ok([region(UNKNOWN_C0,4)?,region(UNKNOWN_C1,8)?])
+    }
+
+    pub(crate) fn stats_region(&self) -> Result<init::Region> {
+        let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended {
+            UNKNOWN_C3
+        } else { HARDWARE_DATA }];
+        init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL)
+    }
+
+    pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice,
+                                  device: &crate::m3::m3_device::Device,
+                                  sequence: u32) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended {
+            let state = device.pstate_register()? & 15;
+            let performance = &mut self.objects[super::m3_init_26::PERFORMANCE_CONTROL];
+            let actual = performance.read_u32(0x2c)?;
+            let target = performance.read_u32(0x30)?;
+            let hardware = performance.read_u32(0x38)?;
+            dev_info!(dev.as_ref(), "M3_NATIVE_POWER sample={} state={} actual={} target={} hardware={}\n",
+                sequence, state, actual, target, hardware);
+            return Ok(());
+        }
+        let cap = self.thermal.allowed(kernel::time::Instant::<kernel::time::Monotonic>::now());
+        self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA],
+                               device.pstate_register()?, cap, sequence)?;
+        type P=crate::m3::g15_raw::raw::PowerCtlBlock;
+        type G=crate::m3::g15_raw::raw::GlobalsG15V14_8_3;
+        let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?;
+        let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?;
+        let power_if=core::mem::offset_of!(G,power_if_target);
+        let target1=self.objects[GLOBALS].read_u32(power_if+4)?;
+        let target2=self.objects[GLOBALS].read_u32(power_if+8)?;
+        let mut override_active=[0];
+        self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?;
+        dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n",
+            sequence,crate::m3::m3_params::consistent_perf(),request,cap_word,target1,target2,
+            self.request_writes,self.request_observed,override_active[0]);
+        Ok(())
+    }
+
+    pub(crate) fn render_pb(&mut self) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); }
+        self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?)
+    }
+    pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result {
+        if kind>2 {return Err(EINVAL);}
+        let state=kind*2;let ring=state+1;
+        let w=self.objects[state].read_u32(0x20)?;
+        if w>=256 || self.objects[state].read_u32(0)?!=w { return Err(EBUSY); }
+        let mut message=[0u8;0x18];
+        message[8..16].copy_from_slice(&queue.to_le_bytes());
+        message[16..20].copy_from_slice(&(kind as u32).to_le_bytes());
+        message[20..22].copy_from_slice(&head.to_le_bytes());
+        message[22]=event;message[23]=u8::from(new); // New queue; caller selected the event slot.
+        self.objects[ring].write(w as usize*0x18,&message)?;
+        agx_memory::publish();
+        self.objects[state].u32(0x20,(w+1)%256)?;
+        agx_memory::publish();
+        Ok(())
+    }
+    /// Whether the firmware has consumed every message on pipe `kind` (0 TA, 1 3D, 2 compute).
+    pub(crate) fn pipe_free(&mut self, kind:usize) -> Result<bool> {
+        if kind>2 {return Err(EINVAL);}
+        Ok(self.objects[kind*2].read_u32(0)? == self.objects[kind*2].read_u32(0x20)?)
+    }
+    pub(crate) fn pipes_idle(&mut self) -> Result<bool> {
+        for state in [0,2,4] {
+            if self.objects[state].read_u32(0)? != self.objects[state].read_u32(0x20)? {
+                return Ok(false);
+            }
+        }
+        Ok(true)
+    }
+    pub(crate) fn ready(&mut self) -> Result<bool> {
+        if self.profile == super::m3_profile::Profile::Extended {
+            return Ok(self.objects[RUNTIME_FLAGS].read_u32(0x14)? == 1);
+        }
+        // NewRegionB+4 is an input initialized to 1, not an acknowledgement.
+        // These are firmware-written transitions present in compute-15's
+        // pre-submit snapshot; both start at zero in the exported schema.
+        Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd
+            && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1)
+    }
+    pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended {
+            let ready = self.objects[RUNTIME_FLAGS].read_u32(0x14)?;
+            let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?;
+            dev_info!(dev.as_ref(), "M3: extended firmware ready={} control-generation={}/{}\n",
+                ready, consumed, self.control_generation);
+            return Ok(());
+        }
+        let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?;
+        let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?;
+        let a=self.objects[RUNTIME_FLAGS].read_u32(4)?;
+        let b=self.objects[RUNTIME_FLAGS].read_u32(0x14)?;
+        let c=self.objects[CONTROL_REGION].read_u32(0x45c4)?;
+        dev_info!(dev.as_ref(),"M3 firmware readiness: magic={:#x} runtime={} newA={}/{} newB={}\n",magic,state,a,b,c);
+        Ok(())
+    }
+    /// The firmware's halt counter and its halted and resume flags.
+    pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> {
+        if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); }
+        let control=&mut self.objects[CONTROL_REGION];
+        Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?))
+    }
+    /// Let a halted firmware continue: clear halted, then set resume. These
+    /// words keep the halt/resume layout of earlier Apple GPU firmware.
+    pub(crate) fn resume_halted(&mut self) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); }
+        let control=&mut self.objects[CONTROL_REGION];
+        control.u32(0x4590,0)?;
+        agx_memory::publish();
+        control.u32(0x45a0,1)?;
+        agx_memory::publish();
+        Ok(())
+    }
+    /// Read-only snapshot of the firmware's recovery words in the control
+    /// region (InitData+0xb0): the halt counter and the halted/resume flags
+    /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information
+    /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0.
+    /// Nothing is acknowledged, cleared or resumed here.
+    pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended {
+            for (owner, start, length) in [(super::m3_init_26::RECOVERY_ACTIVITY, 0, 0x60),
+                (CONTROL_REGION, 0x4ebc, 0x100)] {
+                for offset in (start..start + length).step_by(32) {
+                    let mut bytes = [0; 32];
+                    self.objects[owner].read(offset, &mut bytes)?;
+                    dev_err!(dev.as_ref(), "M3: firmware recovery owner={} offset={:#x} bytes={:02x?}\n",
+                        owner, offset, bytes);
+                }
+            }
+            return Ok(());
+        }
+        let control=&mut self.objects[CONTROL_REGION];
+        let halt_count=control.read_u64(0x4580)?;
+        let halted=control.read_u32(0x4590)?;
+        let resume=control.read_u32(0x45a0)?;
+        let info=[control.read_u32(0x40a0)?,control.read_u32(0x40a4)?];
+        let mut packet=[0u8;0x14];
+        control.read(0x44a0,&mut packet)?;
+        dev_info!(dev.as_ref(),"M3 firmware recovery state: halt_count={} halted={} resume={} info={:#x}/{:#x} packet={:02x?} completed_events={} event_messages={}\n",
+            halt_count,halted,resume,info[0],info[1],packet,self.completed_events,events);
+        Ok(())
+    }
+    /// Read-only: log the event ring entries from the host's read index to the firmware's write
+    /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the
+    /// completion count is not changed.
+    pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result {
+        let size = if self.profile == super::m3_profile::Profile::Extended { 0x48 } else { 0x38 };
+        const COUNT: usize = 256;
+        let r = self.objects[storage::EVENT*2].read_u32(0)? as usize;
+        let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize;
+        dev_info!(dev.as_ref(), "M3 event ring: read={} write={}\n", r, w);
+        if r >= COUNT || w >= COUNT { return Err(EIO); }
+        let mut index = r;
+        for _ in 0..16 {
+            if index == w { break; }
+            let mut raw = [0u8; 0x48];
+            self.objects[storage::EVENT*2+1].read(index * size, &mut raw[..size])?;
+            dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n",
+                index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw);
+            index = (index + 1) % COUNT;
+        }
+        Ok(())
+    }
+    pub(crate) fn control(&mut self, opcode: u32) -> Result<u32> {
+        if !matches!(opcode, 0x13 | 9) { return Err(EINVAL); }
+        let w = self.objects[storage::DEVICE_CONTROL*2].read_u32(0x20)?;
+        let r = self.objects[storage::DEVICE_CONTROL*2].read_u32(0)?;
+        if w >= 256 || r != w { return Err(EBUSY); }
+        let mut bytes = [0; 0x38];
+        bytes[..4].copy_from_slice(&opcode.to_le_bytes());
+        self.objects[storage::DEVICE_CONTROL*2+1].write(w as usize * 0x38, &bytes)?;
+        agx_memory::publish();
+        let next = (w + 1) % 256;
+        self.objects[storage::DEVICE_CONTROL*2].u32(0x20, next)?;
+        agx_memory::publish();
+        Ok(next)
+    }
+    /// Publish an extended-format synchronous control. The caller rings the
+    /// doorbell after this method and waits for both retirement cursors.
+    pub(crate) fn control_extended(&mut self, action: super::m3_channel_format::Action) -> Result<u32> {
+        use super::m3_channel_format as format;
+        if self.profile != super::m3_profile::Profile::Extended { return Err(EINVAL); }
+        let state = storage::DEVICE_CONTROL * 2;
+        let cursors = [self.objects[state].read_u32(0)?,
+            self.objects[state].read_u32(0x10)?, self.objects[state].read_u32(0x20)?];
+        let cursor = format::idle_cursor(cursors).map_err(|error| {
+            if error == init::Error::Bounds { EIO } else { EBUSY }
+        })?;
+        let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?;
+        let generation = action.generation(self.control_generation, consumed).map_err(|_| EOVERFLOW)?;
+        let mut record = [0; format::CONTROL_BYTES];
+        action.encode(&mut record).map_err(|_| EINVAL)?;
+        self.objects[state + 1].write(cursor as usize * format::CONTROL_BYTES, &record)?;
+        if generation != 0 {
+            self.objects[GLOBALS].write(0xe41, &generation.to_le_bytes())?;
+        }
+        agx_memory::publish();
+        let next = (cursor + 1) % format::SLOTS;
+        self.objects[state].u32(0x20, next)?;
+        agx_memory::publish();
+        self.control_generation = generation;
+        Ok(next)
+    }
+
+    pub(crate) fn control_done(&mut self, next: u32) -> Result<bool> {
+        Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next)
+    }
+    pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result {
+        if self.profile == super::m3_profile::Profile::Extended {
+            return self.drain_extended(dev);
+        }
+        // Event, trace, stats and six independent firmware log rings.
+        for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1),
+            (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1),
+            (storage::STATS*2,storage::STATS*2+1,0x40,256,1),
+            (storage::FIRMWARE_LOG*2,storage::FIRMWARE_LOG*2+1,init::fwlog::ENTRY_SIZE,init::fwlog::SLOTS,init::fwlog::CHANNELS)] {
+            for channel in 0..channels {
+                let base = channel * 0x30;
+                let mut r = self.objects[state].read_u32(base)? as usize;
+                let w = self.objects[state].read_u32(base+0x20)? as usize;
+                if r >= count || w >= count { return Err(EIO); }
+                let previous_read = r;
+                while r != w {
+                    if (state==storage::TRACE*2 && self.completed_events<4) || state==storage::FIRMWARE_LOG*2 {
+                        let mut raw=[0u8;0xd8];
+                        self.objects[ring].read((channel*count+r)*size,&mut raw[..size])?;
+                        dev_info!(dev.as_ref(),"M3 fw-channel {} {:02x?}\n",state,&raw[..size]);
+                        if state == storage::FIRMWARE_LOG*2 && self.objects[ring].read_u32((channel*count+r)*size)? == 2 {
+                            let index = self.objects[ring].read_u64((channel*count+r)*size+8)?;
+                            let offset = init::fwlog::payload_offset(channel,index).map_err(|_| EIO)?;
+                            let mut payload = [0u8;init::fwlog::PAYLOAD_SIZE];
+                            self.objects[FWLOG_PAYLOAD].read(offset,&mut payload)?;
+                            dev_info!(dev.as_ref(),"M3 fwlog payload {} {:02x?}\n",channel,payload);
+                        }
+                    }
+                    if state == storage::EVENT*2 {
+                        let kind = self.objects[ring].read_u32(r*size)?;
+                        if kind == 1 {
+                            let flags=self.objects[ring].read_u64(r*size+4)?;
+                            self.completed_events+=u64::from((flags&7).count_ones());
+                        }
+                        if kind == 0 || kind == 4 {
+                            let mut raw=[0u8;0x38];self.objects[ring].read(r*size,&mut raw)?;
+                            dev_err!(dev.as_ref(), "M3 firmware error event {} bytes={:02x?}\n", kind,raw);
+                            return Err(EIO);
+                        }
+                    }
+                    r = (r+1) % count;
+                }
+                // Publish actual consumption only, as RxChannel::get does.
+                // Empty polls need no write to firmware's shared state line.
+                if r != previous_read { self.objects[state].u32(base, r as u32)?; }
+            }
+        }
+        Ok(())
+    }
+    fn drain_extended(&mut self, dev: &driver::AsahiDrmDevice) -> Result {
+        use super::m3_channel_format as format;
+        // Snapshot each producer once. New arrivals remain for the next drain;
+        // a corrupt cursor is rejected before reading its ring allocation.
+        for (channel, capacity, threads) in [(storage::FIRMWARE_LOG, 256, 9),
+            (storage::EVENT, 256, 1), (storage::TRACE, 512, 1), (storage::STATS, 256, 1)] {
+            let state = channel * 2;
+            for thread in 0..threads {
+                let at = thread * 0x30;
+                let mut read = self.objects[state].read_u32(at)?;
+                let write = self.objects[state].read_u32(at + 0x20)?;
+                if read >= capacity || write >= capacity { return Err(EIO); }
+                let before = read;
+                while read != write {
+                    let offset = (thread * capacity as usize + read as usize) * format::RECEIVE_BYTES;
+                    let mut record = [0; format::RECEIVE_BYTES];
+                    self.objects[state + 1].read(offset, &mut record)?;
+                    let kind = u32::from_le_bytes(record[..4].try_into().map_err(|_| EIO)?);
+                    if channel == storage::FIRMWARE_LOG {
+                        let slot = u32::from_le_bytes(record[8..12].try_into().map_err(|_| EIO)?);
+                        let payload = format::payload_offset(thread, slot).map_err(|_| EIO)?;
+                        if crate::debug::debug_enabled(crate::debug::DebugFlags::KTraceCh) {
+                            let mut text = [0; 200];
+                            self.objects[FWLOG_PAYLOAD].read(payload + 0x10, &mut text)?;
+                            let len = text.iter().position(|value| *value == 0).unwrap_or(text.len());
+                            dev_info!(dev.as_ref(), "M3 firmware log[{}]: {}\n", thread,
+                                core::str::from_utf8(&text[..len]).unwrap_or("<non-UTF8>"));
+                        }
+                    } else if channel == storage::EVENT {
+                        if kind != 1 {
+                            dev_err!(dev.as_ref(), "M3 firmware event {}: {:02x?}\n", kind, record);
+                            return Err(EIO);
+                        }
+                        let flags = u32::from_le_bytes(record[4..8].try_into().map_err(|_| EIO)?);
+                        self.completed_events = self.completed_events.saturating_add((flags & 7).count_ones().into());
+                    } else if (channel == storage::TRACE && kind != 5)
+                        || (channel == storage::STATS && kind == 0x16) {
+                        dev_err!(dev.as_ref(), "M3 firmware channel {} record {}: {:02x?}\n", channel, kind, record);
+                        return Err(EIO);
+                    }
+                    read = (read + 1) % capacity;
+                }
+                if read != before { self.objects[state].u32(at, read)?; }
+            }
+        }
+        agx_memory::publish();
+        Ok(())
+    }
+
+}
+
+/// All extended initialization owners, retained as one construction transaction.
+/// Nothing is published to RTKit by this constructor. A failed allocation drops
+/// its mappings before backing storage and leaves no root visible to firmware.
+pub(crate) struct ExtendedStorage {
+    pub(crate) objects: KVec<Buffer>,
+    pub(crate) iomaps: KVec<mmu::KernelMapping>,
+    pub(crate) descriptors: KVec<u8>,
+    pub(crate) maximum_khz: u32,
+}
+
+impl ExtendedStorage {
+    pub(crate) fn new(
+        dev: &driver::AsahiDrmDevice,
+        uat: &mmu::Uat,
+        pwr: &super::hw::PwrConfig,
+        sram_uv: &[u32],
+        bases: super::m3_mmio_26::RegisterBases,
+    ) -> Result<Self> {
+        use super::{m3_init_26 as plan, m3_mmio_26 as io};
+        let mut objects = KVec::new();
+        for index in 0..plan::COUNT {
+            let entry = plan::allocation(index).map_err(|_| EINVAL)?;
+            let alias = match index {
+                plan::GLOBAL_POOL => Some((uat.kernel_lower_vm(), 0x70_0000_0000)),
+                plan::GLOBAL_PARAMETERS => Some((uat.kernel_lower_vm(), 0x70_0000_8000)),
+                _ => None,
+            };
+            let object = if plan::cached(index) {
+                Buffer::at_prot(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size,
+                    prot::PROT_FW_PRIV_RW, prot::PROT_GPU_SHARED_RW)?
+            } else {
+                Buffer::at(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size, entry.gpu_shared)?
+            };
+            objects.push(object, GFP_KERNEL)?;
+        }
+        let mut descriptors = KVec::new();
+        descriptors.resize(io::SLOTS * io::RECORD_BYTES, 0, GFP_KERNEL)?;
+        let mut iomaps = KVec::new();
+        for slot in 0..io::SLOTS {
+            let mut region = None;
+            if let Some(area) = io::area(slot, bases).map_err(|_| EINVAL)? {
+                let address = 0xffff_fc23_0000_0000 + slot as u64 * 0x10_0000;
+                let access = if area.writable() { prot::PROT_FW_MMIO_RW } else { prot::PROT_FW_MMIO_RO };
+                for (physical, bytes, offset) in area.banks() {
+                    if bytes == 0 { continue; }
+                    let mapping = uat.kernel_vm().map_io_m3(address + offset as u64,
+                        physical.try_into()?, bytes, access)?;
+                    iomaps.push(mapping, GFP_KERNEL)?;
+                }
+                region = Some(init::Region::new(address, area.mapped_bytes()).map_err(|_| EINVAL)?);
+            }
+            io::encode(slot, bases, region,
+                &mut descriptors[slot * io::RECORD_BYTES..(slot + 1) * io::RECORD_BYTES])
+                .map_err(|_| EINVAL)?;
+        }
+        // Runtime+0x18 references Fender SRAM, rather than anonymous DMA RAM.
+        // Retain its normal shared mapping independently of its MMIO view.
+        let scratch_physical = bases.gpu.checked_add(0xd6_0000).ok_or(EOVERFLOW)?;
+        let scratch = uat.kernel_vm().map_io_m3(0xffff_fc22_0000_0000,
+            scratch_physical.try_into()?, 0x8000, prot::PROT_FW_SHARED_RW)?;
+        let scratch_region = init::Region::new(scratch.iova(), 0x8000).map_err(|_| EINVAL)?;
+        iomaps.push(scratch, GFP_KERNEL)?;
+        let mut regions = [init::Region::new(0xffff_fc21_0000_0000, 1).map_err(|_| EINVAL)?; plan::COUNT];
+        for (index, object) in objects.iter().enumerate() {
+            regions[index] = init::Region::new(object.va(), object.size()).map_err(|_| EINVAL)?;
+        }
+        let graph = plan::PointerGraph { owners: &regions, scratch: scratch_region,
+            pool_alias: objects[plan::GLOBAL_POOL].gpu_va()?,
+            parameter_alias: objects[plan::GLOBAL_PARAMETERS].gpu_va()? };
+        let mut root = [0; 0xc8];
+        let mut runtime = [0; plan::RUNTIME_SIZE];
+        let mut control = KVec::new();
+        control.resize(plan::allocation(CONTROL_REGION).map_err(|_| EINVAL)?.size, 0, GFP_KERNEL)?;
+        graph.root(&mut root).map_err(|_| EINVAL)?;
+        graph.runtime(&mut runtime).map_err(|_| EINVAL)?;
+        graph.firmware_control(&mut control).map_err(|_| EINVAL)?;
+        let images = super::m3_power_26::PowerImages::new(pwr, sram_uv,
+            objects[plan::SENSOR_VALUE].va(), objects[plan::RETENTION_STATE].va(),
+            agx_memory::TIMESTAMP_RANGE.start, &descriptors)?;
+        objects[storage::ROOT].write(0, &root)?;
+        objects[RUNTIME_POINTERS].write(0, &runtime)?;
+        objects[CONTROL_REGION].write(0, &control)?;
+        objects[HARDWARE_DATA].write(0, &images.hardware)?;
+        objects[plan::PERFORMANCE_CONTROL].write(0, &images.performance)?;
+        objects[GLOBALS_POWER].write(0, &images.power)?;
+        objects[GLOBALS].write(0, &images.globals)?;
+        // Fragment scheduler entries start unassigned; these are input sentinels.
+        objects[UNKNOWN_C1].u32(0xc18, u32::MAX)?;
+        objects[UNKNOWN_C1].u32(0xc30, u32::MAX)?;
+        agx_memory::publish();
+        Ok(Self { objects, iomaps, descriptors, maximum_khz: images.maximum_khz })
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs
new file mode 100644
index 0000000000000..9897690d923dc
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_device.rs
@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Owned M3 register apertures. The platform
+//! power domain owns TVM/PMGR and stays on across probe/remove. Firmware and
+//! queues must be stopped before this object is released.
+
+use kernel::{
+    bindings, c_str,
+    device::Core,
+    error::to_result,
+    io::{
+        mem::DevresIoMem,
+        Io,
+        IoBase,
+    },
+    platform,
+    prelude::*,
+    sync::aref::ARef,
+    time::{delay::fsleep, Delta, Instant, Monotonic},
+};
+
+use crate::m3::m3_firmware::Firmware;
+use crate::m3::m3_soc::Soc;
+
+const ASC_CPU_CONTROL: usize = 0x44;
+const ASC_CPU_RUN: u32 = 1 << 4;
+
+pub(crate) struct Device {
+    dev: ARef<platform::Device>,
+    asc: DevresIoMem<0x4000>,
+    sgx: DevresIoMem,
+    firmware: Firmware,
+    core_mask: u64,
+    soc: &'static Soc,
+    power_vote: bool,
+}
+
+impl Device {
+    /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides
+    /// the SGX setup write, which the T8122 table lacks.
+    pub(crate) fn new(pdev: &platform::Device<Core<'_>>, firmware: Firmware, soc: &'static Soc,
+        experiment: Option<&crate::m3::t8122_start::Experiment>) -> Result<Self> {
+        // Map ASC before taking a vote. The larger SGX aperture contains
+        // this control window and the separate mailbox provider, so it is
+        // mapped without a conflicting claim over those child resources.
+        // Only the 16 KiB control window is used. Some device trees describe
+        // the ASC block as a larger range that also covers the mailbox, which
+        // the mailbox driver has already claimed, so do not claim the range.
+        let asc = pdev.io_request_by_name(c_str!("asc"))
+            .ok_or(EINVAL)?
+            .iomap_sized::<0x4000>()?
+            .into_devres()?;
+        dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n");
+        let sgx = pdev.io_request_by_name(c_str!("sgx"))
+            .ok_or(EINVAL)?
+            .iomap()?
+            .into_devres()?;
+        dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n");
+        let pmp = crate::m3::m3_board::has_pmp_link(pdev);
+        let t8122 = crate::m3::t8122_start::is_t8122(soc);
+        if pmp && !t8122 {
+            dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n");
+            return Err(ENOTSUPP);
+        }
+        if t8122 && !pmp {
+            dev_err!(pdev.as_ref(), "M3 G15G: current14 GPU requires its PMP supplier link\n");
+            return Err(ENODEV);
+        }
+        if pmp {
+            // SAFETY: the consumer is live; the managed link pins its declared PMP supplier.
+            to_result(unsafe { bindings::apple_pmp_link_device(pdev.as_ref().as_raw()) })?;
+        } else {
+            dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n");
+        }
+        if asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 {
+            dev_err!(
+                pdev.as_ref(),
+                "M3 {}: ASC already running; refusing to take ownership\n",
+                soc.gpu_name
+            );
+            return Err(EBUSY);
+        }
+        let mut device = Self {
+            dev: pdev.into(),
+            asc,
+            sgx,
+            firmware,
+            core_mask: 0,
+            soc,
+            power_vote: false,
+        };
+
+        if pmp {
+            // SAFETY: the linked current14 T8122 PMP owns AGX logical device 5. The bridge
+            // waits for firmware readiness and the power-command acknowledgement.
+            // An unacknowledged request may still have enabled AGX. Keep ownership so a
+            // failed acquisition releases it only after the ASC-stopped check in Drop.
+            device.power_vote = true;
+            to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?;
+            dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n");
+        }
+
+        let registers = device.sgx.access(pdev.as_ref())?.as_view();
+        let version = registers.try_read32(0xd04000)?;
+        let counts = registers.try_read32(0xd04010)?;
+        // The core-enable words: the second only on a SoC with more than 32 core slots.
+        let mut core_mask = u64::from(registers.try_read32(0xe01500)?);
+        if crate::m3::m3_board::core_slots(soc) > 32 {
+            core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32;
+        }
+        dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n",
+            soc.gpu_name, version, counts, core_mask, device.firmware.version());
+        // These words identify the qualified configuration of this SoC. The
+        // fused core mask varies with SKU; absent cores must never be enabled.
+        if version & soc.id.version_mask != soc.id.version
+            || counts & soc.id.counts_mask != soc.id.counts
+            || !crate::m3::m3_board::core_mask_valid(soc, core_mask)
+        {
+            return Err(ENODEV);
+        }
+        let setup = match experiment {
+            Some(e) => {
+                match e.sgx_setup() {
+                    Some((offset, value)) => dev_warn!(pdev.as_ref(),
+                        "M3 G15G start: SGX setup write: SGX+{:#x} = {:#x} (asahi.t8122_sgx_setup=t6030)\n", offset, value),
+                    None => dev_info!(pdev.as_ref(),
+                        "M3 G15G start: no SGX setup write (asahi.t8122_sgx_setup=none)\n"),
+                }
+                e.sgx_setup()
+            }
+            None if device.firmware.profile() == super::m3_profile::Profile::Extended => None,
+            None => Some(soc.sgx_setup.ok_or(ENODEV)?),
+        };
+        if let Some((setup_offset, setup_value)) = setup {
+            registers.try_write32(setup_value, setup_offset)?;
+        }
+        device.core_mask = core_mask;
+        Ok(device)
+    }
+
+    pub(crate) fn core_mask(&self) -> u64 { self.core_mask }
+
+    /// The SoC table this device was admitted with.
+    pub(crate) fn soc(&self) -> &'static Soc { self.soc }
+
+    pub(crate) fn check_idle(&self)->Result {
+        self.check_idle_parts(true,true)
+    }
+    /// [`Self::check_idle`] with its engine-busy (`busy`) and fault-bank (`faults`) register
+    /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register.
+    pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result {
+        if !busy && !faults {return Ok(());}
+        let sgx_guard=self.sgx.try_access().ok_or(ENODEV)?;
+        let sgx=sgx_guard.as_view();
+        if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 {
+            return Err(EBUSY);
+        }
+        if !faults {return Ok(());}
+        let selector=sgx.try_read64(0xd800)?;
+        let mut fault=0;
+        // One fault bank (selected through 0xd800) per GPU cluster: banks 0 and 1 on the
+        // two-cluster T6030, bank 0 alone on the single-cluster T8122. Never select a bank past
+        // the SoC's clusters.
+        for bank in 0..u64::from(self.soc.clusters) {
+            sgx.try_write64(bank,0xd800)?;fault|=sgx.try_read64(0xd8c0)?;
+        }
+        sgx.try_write64(selector,0xd800)?;
+        if fault!=0 {return Err(EIO);} Ok(())
+    }
+
+    /// The raw performance-state register (SGX+0xe01000).
+    pub(crate) fn pstate_register(&self) -> Result<u32> {
+        self.sgx.try_access().ok_or(ENODEV)?.as_view().try_read32(0xe01000)
+    }
+
+    /// Read-only snapshot. Do not select or acknowledge fault banks here;
+    /// the firmware owns those registers until recovery is ported.
+    pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result {
+        if self.firmware.profile() == super::m3_profile::Profile::Extended {
+            // Adaptive clock gating can leave individual engine banks inaccessible
+            // even while the GPU performance-state register says powered up.
+            return Ok(());
+        }
+        let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?;
+        let sgx = sgx_guard.as_view();
+        let pstate = sgx.try_read32(0xe01000)? & 0xf;
+        if pstate == 0 {
+            dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name);
+            return Ok(());
+        }
+        dev_info!(self.dev.as_ref(), "M3 {}: reading engine/MMU snapshot at pstate={}\n", self.soc.gpu_name, pstate);
+        let selector=sgx.try_read64(0xd800)?;
+        for bank in [0,1,0x100,0x40000,0x40001,0x80000,0xc0000] {
+            sgx.try_write64(bank,0xd800)?;
+            let address=sgx.try_read64(0xd8c8)?;
+            let fault=sgx.try_read64(0xd8c0)?;
+            dev_info!(self.dev.as_ref(),"M3 bank {} fault={:#x} address_word={:#x} address={:#x}\n",bank,fault,address,address<<6);
+            if fault&1!=0 {
+                dev_info!(self.dev.as_ref(),"M3 fault mapping bank={} address={:#x} translation={:?}\n",bank,address<<6,vm.translate_iova(address<<6));
+            }
+        }
+        sgx.try_write64(selector,0xd800)?;
+        // Read-only per-core service/debug snapshot, using the same
+        // core-selector protocol as the idle check.
+        let service=sgx.try_read32(0xa010)?;
+        let debug=sgx.try_read32(0xa000)?;
+        let mut cores=u64::from(sgx.try_read32(0xe01500)?);
+        if crate::m3::m3_board::core_slots(self.soc) > 32 {
+            cores |= u64::from(sgx.try_read32(0xe01504)?) << 32;
+        }
+        for core in 0..crate::m3::m3_board::core_slots(self.soc).min(64) {
+            if cores & (1u64<<core)==0 {continue;}
+            sgx.try_write32(core,0xa010)?;
+            sgx.try_write32(core,0xa000)?;
+            dev_info!(self.dev.as_ref(),"M3_USC core={} VDM={:#x}/{:#x} PDM={:#x}/{:#x} CDM={:#x}/{:#x}\n",
+                core,sgx.try_read64(0xa088)?,sgx.try_read64(0xa090)?,
+                sgx.try_read64(0xa068)?,sgx.try_read64(0xa070)?,
+                sgx.try_read64(0xa0a8)?,sgx.try_read64(0xa0b0)?);
+        }
+        sgx.try_write32(service,0xa010)?;
+        sgx.try_write32(debug,0xa000)?;
+        let mut mmu = [0u32; 13];
+        for (i, word) in mmu.iter_mut().enumerate() { *word = sgx.try_read32(0xd08000 + i * 4)?; }
+        dev_info!(self.dev.as_ref(), "M3 {}: GPU MMU configuration={:x?}\n", self.soc.gpu_name, mmu);
+        for offset in [0xc000,0xc008,0xc010,0xc018,0xc020,0xc028,0xc030,0xc038,0xc060,0xc068,0xc070,0xc078,0xc088,0xc090,0xc098,0xc0a0,0xc0a8,0xc0b0, 0xc040, 0xc048, 0xc050, 0xc058, 0xc080, 0xc120, 0xc140, 0xc148,
+            0xd8c0, 0xd8c8] {
+            dev_info!(self.dev.as_ref(), "M3 {}: SGX +{:#x}={:#018x}\n", self.soc.gpu_name, offset, sgx.try_read64(offset)?);
+        }
+        Ok(())
+
+    }
+
+    pub(crate) fn firmware(&self) -> &Firmware { &self.firmware }
+
+    pub(crate) fn require_stopped(&self, pdev: &platform::Device<Core<'_>>) -> Result {
+        if self.asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 {
+            return Err(EBUSY);
+        }
+        Ok(())
+    }
+
+    pub(crate) fn start_asc(&self, pdev: &platform::Device<Core<'_>>) -> Result {
+        self.require_stopped(pdev)?;
+        let asc = self.asc.access(pdev.as_ref())?.as_view();
+        asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL);
+        Ok(())
+    }
+
+    pub(crate) fn stop_asc(&self) -> Result {
+        let asc_guard = self.asc.try_access().ok_or(ENODEV)?;
+        let asc = asc_guard.as_view();
+        asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL);
+        let start = Instant::<Monotonic>::now();
+        while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 {
+            if start.elapsed() >= Delta::from_millis(100) { return Err(ETIMEDOUT); }
+            fsleep(Delta::from_micros(10));
+        }
+        Ok(())
+    }
+
+}
+
+impl Drop for Device {
+    fn drop(&mut self) {
+        if !self.power_vote {
+            return;
+        }
+        // ASC and firmware queues must be stopped before the inner GPU vote is relinquished.
+        match self.asc.try_access() {
+            Some(asc) if asc.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {}
+            _ => {
+                dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n");
+                return;
+            }
+        }
+        // SAFETY: the managed consumer link keeps the PMP bound through device removal.
+        if let Err(error) = to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 0) }) {
+            dev_err!(self.dev.as_ref(), "M3 G15G: PMP GPU power release failed: {:?}\n", error);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs
new file mode 100644
index 0000000000000..b11b612184ed9
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! M3 through the common Asahi GEM, GPUVM and render-node frontend.
+use core::ops::Range;
+use core::any::Any;
+use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}, uapi};
+use crate::{alloc, driver, gem, gpu, m3::hw, mmu, queue};
+
+pub(crate) type Shared = Arc<Mutex<Option<crate::m3::m3_runtime::Runtime>>>;
+struct ProgressView(Arc<crate::m3::m3_rtkit::Health>);
+impl kernel::debugfs::Writer for ProgressView {
+    fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        let (generation, completed, last_ns, healthy) = self.0.progress_snapshot();
+        writeln!(f, "version=1 generation_ns={} completed={} last_completion_ns={} healthy={}",
+            generation, completed, last_ns, u8::from(healthy))
+    }
+}
+// Separate endpoint preserves the strict completion-v1 ABI for old guards.
+struct ActivityView(Arc<crate::m3::m3_rtkit::Health>);
+impl kernel::debugfs::Writer for ActivityView {
+    fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        let (generation, epoch, healthy) = self.0.activity_snapshot();
+        writeln!(f, "version=1 generation_ns={} epoch={} pending={} healthy={}",
+            generation, epoch, epoch & 1, u8::from(healthy))
+    }
+}
+
+struct TimingView(Arc<crate::m3::m3_rtkit::Health>);
+impl kernel::debugfs::Writer for TimingView {
+    fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        let (generation, _, _, healthy) = self.0.progress_snapshot();
+        self.0.timing.write(f, generation, healthy)
+    }
+}
+
+// Root-only writable debugfs policy; no mutable module-parameter references.
+// Keep ownership independent of the runtime lock and remove the file on stop.
+struct CompletionView(Arc<crate::m3::m3_rtkit::EventWait>);
+impl kernel::debugfs::Writer for CompletionView {
+    fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        let (us, irqs, sleeps, timers, preparations) = self.0.snapshot();
+        writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}",
+                 us, irqs, sleeps, timers, preparations)
+    }
+}
+impl kernel::debugfs::Reader for CompletionView {
+    fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result {
+        let len = reader.len();
+        if len == 0 || len > 8 { return Err(EINVAL); }
+        let mut bytes = [0u8; 8];
+        reader.read_slice(&mut bytes[..len])?;
+        let text = core::str::from_utf8(&bytes[..len]).map_err(|_| EINVAL)?;
+        let us = text.trim().parse::<u64>().map_err(|_| EINVAL)?;
+        self.0.set_wait_us(us)
+    }
+}
+
+struct SubmitOverlapView(Arc<crate::m3::m3_rtkit::EventWait>);
+impl kernel::debugfs::Writer for SubmitOverlapView {
+    fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        writeln!(f, "{}", u8::from(self.0.cpu_overlap()))
+    }
+}
+impl kernel::debugfs::Reader for SubmitOverlapView {
+    fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result {
+        let len=reader.len();
+        if len==0 || len>2 {return Err(EINVAL);}
+        let mut bytes=[0u8;2];reader.read_slice(&mut bytes[..len])?;
+        let text=core::str::from_utf8(&bytes[..len]).map_err(|_|EINVAL)?.trim();
+        match text {
+            "0" => self.0.set_cpu_overlap(false),
+            "1" => self.0.set_cpu_overlap(true),
+            _ => return Err(EINVAL),
+        }
+        Ok(())
+    }
+}
+
+/// The started T8122 runtime, as the shared DRM layer sees it ([`gpu::Gpu`]). It is created at
+/// probe with the runtime it wraps and handed to the DRM registration, which owns it: dropping it
+/// (unbind, or a failed registration) stops the runtime. The debugfs files share its lifetime.
+pub(crate) struct Backend {
+    shared: Shared,
+    health: Arc<crate::m3::m3_rtkit::Health>,
+    ids: gpu::SequenceIDs,
+    scheduler: Arc<drm::sched::Scheduler<crate::m3::m3_submit::Job>>,
+    core_mask: u64,
+    max_frequency_khz: u32,
+    soc: &'static crate::m3::m3_soc::Soc,
+    profile: super::m3_profile::Profile,
+    _progress: Pin<KBox<kernel::debugfs::File<ProgressView>>>,
+    _activity: Pin<KBox<kernel::debugfs::File<ActivityView>>>,
+    _memory: Pin<KBox<kernel::debugfs::File<crate::m3::agx_memory_stats::View>>>,
+    _timing: Pin<KBox<kernel::debugfs::File<TimingView>>>,
+    _completion: Pin<KBox<kernel::debugfs::File<CompletionView>>>,
+    _overlap: Pin<KBox<kernel::debugfs::File<SubmitOverlapView>>>,
+}
+
+impl Backend {
+    /// HIGH promises host publication ordering, not firmware preemption.
+    pub(crate) fn supports_high_priority(&self) -> bool {
+        self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended
+    }
+
+    /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the
+    /// caller registers afterwards.
+    pub(crate) fn start(
+        pdev: &platform::Device<Core<'_>>,
+        drm: &driver::AsahiDrmDevice,
+        soc: &'static crate::m3::m3_soc::Soc,
+    ) -> Result<Arc<Self>> {
+        // The runtime only exists to back its render node: without one (`asahi.m3_expose=0`)
+        // nothing is started, rather than leaving a running firmware behind a failed probe.
+        if !crate::m3::m3_board::expose_render_node(soc) {
+            dev_info!(pdev.as_ref(), "M3: render node not registered on this board (asahi.m3_expose=1 registers it); GPU not started\n");
+            return Err(ENODEV);
+        }
+        let resources = crate::m3::m3_resources::from_device(pdev, soc).inspect_err(|_| {
+            crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc);
+        })?;
+        dev_info!(pdev.as_ref(), "M3: resource admission complete\n");
+        // Extended T8122 uses its dedicated initialization graph. Keep the
+        // legacy guarded-start and completeness checks for older interfaces.
+        let extended=crate::m3::t8122_start::is_t8122(soc)
+            && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended;
+        let experiment = if extended {None} else {crate::m3::t8122_start::arm(pdev.as_ref(), soc)?};
+        if experiment.is_none() && !extended {
+            soc.require_complete(pdev.as_ref())?;
+        }
+        let experiment = experiment.as_ref();
+        let refused = |stage: &'static str| {
+            move |e: &Error| crate::m3::t8122_start::refused(pdev.as_ref(), experiment, stage, *e)
+        };
+        let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment)
+            .inspect_err(refused("the firmware identity"))?;
+        dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n");
+        let profile = firmware.profile();
+        // InitData source and contents, before any GPU register access.
+        let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment)
+            .inspect_err(refused("the InitData contents"))?;
+        let device = crate::m3::m3_device::Device::new(pdev, firmware, soc, experiment)
+            .inspect_err(refused("the GPU registers and identity"))?;
+        let core_mask = device.core_mask();
+        let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz();
+        let mut runtime = crate::m3::m3_runtime::Runtime::new(pdev, drm, device, contents)?;
+        runtime.boot(pdev)?;
+        dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n",
+            u8::from(crate::m3::m3_params::unlocked_wait()), u8::from(crate::m3::m3_params::timeout_nohang()),
+            u8::from(crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault)));
+        let health = runtime.health();
+        let state = runtime.rtkit_state();
+        // Health holds no device references. The read-only files own an Arc and are removed
+        // with the backend; teardown latches failure first.
+        let directory = kernel::debugfs::Dir::new(c"asahi-m3");
+        let progress = KBox::pin_init(directory.read_only_file(c"progress",
+            ProgressView(health.clone())), GFP_KERNEL)?;
+        let activity = KBox::pin_init(directory.read_only_file(c"activity",
+            ActivityView(health.clone())), GFP_KERNEL)?;
+        let memory = KBox::pin_init(directory.read_only_file(c"memory",
+            crate::m3::agx_memory_stats::View), GFP_KERNEL)?;
+        let timing = KBox::pin_init(directory.read_only_file(c"timing",
+            TimingView(health.clone())), GFP_KERNEL)?;
+        let completion = KBox::pin_init(directory.read_write_file(c"completion_wait",
+            CompletionView(runtime.completion_wait())), GFP_KERNEL)?;
+        let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap",
+            SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?;
+        let scheduler = Arc::new(drm::sched::Scheduler::new(drm.as_ref().as_ref(), 4, 8, 0, 3000,
+            kernel::c_str!("asahi_m3_sched"))?, GFP_KERNEL)?;
+        let shared: Shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?;
+        let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?;
+        // From here on every failure drops the backend, which stops the runtime.
+        let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(),
+            scheduler, core_mask, max_frequency_khz, soc, profile,
+            _progress: progress, _activity: activity, _memory: memory, _timing: timing,
+            _completion: completion, _overlap: overlap }, GFP_KERNEL)?;
+        // The completion worker services this runtime from now on; the RTKit callbacks and
+        // the submission path queue it through the runtime's state.
+        if !state.completion.populate(completion_work) {
+            return Err(EBUSY);
+        }
+        Ok(backend)
+    }
+
+    /// Stops the runtime: called on unbind, while the device's resources are still live, and
+    /// again (doing nothing) when the backend is dropped.
+    pub(crate) fn stop(&self) {
+        self.health.mark_failed();
+        drop(self.shared.lock().take());
+    }
+}
+
+impl Drop for Backend {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+impl gpu::Gpu for Backend {
+    fn stats_snapshot(&self) -> Option<Arc<crate::stats::StatsSnapshot>> { None }
+    fn as_any(&self) -> &dyn Any { self }
+    fn arc_as_any(self: Arc<Self>) -> Arc<dyn Any + Sync + Send> { self as Arc<dyn Any + Sync + Send> }
+    fn manager(&self) -> Option<&dyn gpu::GpuManager> { None }
+    fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } }
+    fn ids(&self) -> &gpu::SequenceIDs { &self.ids }
+    fn is_crashed(&self) -> bool { !self.health.healthy() }
+    fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result {
+        let soc = self.soc;
+        let masks = crate::m3::m3_board::core_masks(soc, self.core_mask);
+        params.gpu_generation = hw::GpuGen::G15 as u32;
+        params.gpu_variant = soc.gpu_variant as u32;
+        params.gpu_revision = soc.gpu_revision as u32;
+        params.chip_id = soc.chip_id;
+        params.num_dies = 1;
+        params.num_clusters_total = soc.clusters;
+        params.num_cores_per_cluster = soc.cores_per_cluster;
+        for (out, mask) in params.core_masks.iter_mut().zip(masks.iter()) {
+            *out = (*mask).into();
+        }
+        params.max_frequency_khz = self.max_frequency_khz;
+        params.usc_generation = 3;
+        params.gpu_hal_generation = self.profile.hal();
+        params.max_commands_per_submission = super::agx_uapi::UAPI_MAX_HARDWARE_COMMANDS;
+        params.max_attachments = super::agx_uapi::UAPI_MAX_ATTACHMENTS as u32;
+        Ok(())
+    }
+    fn supports_vm_status(&self) -> bool { true }
+    /// The features the 7.1.12 file layer reported for the M3 runtime: VM status (not its
+    /// mirror), scheduled queues, and the SoC's optional render and compute features.
+    fn feature_flags(&self) -> u64 {
+        let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64
+            | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64;
+        if self.profile == super::m3_profile::Profile::Legacy
+            && self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() {
+            flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64;
+        }
+        if self.profile == super::m3_profile::Profile::Legacy
+            && self.soc.features.compute_wide_visibility {
+            flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64;
+        }
+        flags
+    }
+    fn base_clock_hz(&self) -> u32 {
+        let value: u64;
+        // SAFETY: read-only architectural counter frequency.
+        unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) };
+        value as u32
+    }
+    fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } }
+    fn user_range(&self) -> Range<u64> {
+        mmu::UAT_PGSZ as u64..self.profile.user_end()
+    }
+    /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the
+    /// 42-bit geometry's own unk_page() is a different address.
+    fn unknown_page(&self) -> u64 {
+        self.profile.user_end()
+    }
+    fn tracked_submission(&self) -> bool { true }
+    /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate).
+    fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result {
+        crate::m3::m3_client::admit_vm_for(dev, self.profile, params_size, flags)
+    }
+    fn new_vm(&self, range: Range<u64>) -> Result<mmu::Vm> {
+        Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range)
+    }
+    fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc<Mutex<alloc::DefaultAllocator>>,
+        _ualloc_priv: Arc<Mutex<alloc::DefaultAllocator>>, priority: u32,
+        usc_exec_base: u64, _usage: Arc<crate::usage::Client>) -> Result<KBox<dyn queue::Queue>> {
+        Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base,
+            self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64 != 0,
+            self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64 != 0)?,GFP_KERNEL)?)
+    }
+    fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range<usize>) -> Result<crate::timestamp::Buffer> {
+        crate::timestamp::Buffer::firmware(
+            Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range)?)
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs
new file mode 100644
index 0000000000000..94a2d3384d08a
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs
@@ -0,0 +1,593 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+
+use crate::m3::m3_resources as agx_resources;
+
+/// A GPU firmware image the runtime can identify.
+#[derive(Debug)]
+pub(crate) struct KnownImage {
+    /// Human-readable identity, for logs.
+    pub(crate) name: &'static str,
+    uuid: Option<[u8; 16]>,
+    stkg_sha256: Option<[u8; 32]>,
+    /// The validated InitData magic (InitData+0), or `None` until runtime admission is supported.
+    pub(crate) initdata_magic: Option<u64>,
+}
+
+impl KnownImage {
+    /// Runtime admission requires a validated InitData version and all recorded identities.
+    pub(crate) fn accepts_runtime(&self, uuid: Option<[u8; 16]>, digest: &[u8; 32]) -> bool {
+        self.initdata_magic.is_some()
+            && (self.uuid.is_some() || self.stkg_sha256.is_some())
+            && self.uuid.map_or(true, |known| uuid == Some(known))
+            && self.stkg_sha256.map_or(true, |known| known == *digest)
+    }
+}
+
+/// T6030 GPU firmware images known to the M3 runtime. An image is accepted when every recorded
+/// identity (UUID, hash) matches.
+pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [
+    KnownImage {
+        name: "J514S RTKit-2419.140.12",
+        uuid: None,
+        stkg_sha256: Some(crate::m3::m3_firmware::TEXT_SHA256),
+        initdata_magic: Some(G15_V14_8_3_INITDATA),
+    },
+    KnownImage {
+        name: "g15s build b0 (firmware 14.8.3) RTKit-2419.140.12",
+        uuid: Some([
+            0xdb, 0xf3, 0x7c, 0x40, 0xea, 0xd5, 0x37, 0x60, 0x94, 0x41, 0x99, 0x50,
+            0x18, 0x78, 0x29, 0x55,
+        ]),
+        stkg_sha256: None,
+        initdata_magic: Some(G15_V14_8_3_INITDATA),
+    },
+];
+
+/// The G15 InitData version of firmware-compat 14.8.3: the version of both T6030 images and of
+/// the T8122 image.
+pub(crate) const G15_V14_8_3_INITDATA: u64 = 0x0c08_e21e_8380_0490;
+
+/// The image-info UUID of the T8122 C0 firmware-compat 14.8.3 image.
+const T8122_C0_UUID: [u8; 16] = [
+    0xdf, 0x69, 0x7f, 0x05, 0xf6, 0xb5, 0x33, 0xef, 0xa1, 0x37, 0x61, 0xc4, 0xa7, 0x3d, 0x66, 0x6a,
+];
+
+/// T8122 C0, firmware-compat 14.8.3: the image the macOS 14.8.3 system firmware loads on the
+/// M3 MacBook Airs. It takes the G15 14.8.3 InitData version.
+pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage {
+    name: "T8122 C0 firmware-compat 14.8.3",
+    uuid: Some(T8122_C0_UUID),
+    stkg_sha256: None,
+    initdata_magic: Some(G15_V14_8_3_INITDATA),
+}];
+
+/// Exact normalized image identity for the extended firmware interface.
+pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage {
+    name: "T8122 C0 firmware-compat 26.6.2",
+    uuid: None,
+    stkg_sha256: Some([
+        0x3a, 0x6b, 0x8b, 0x5e, 0x83, 0x38, 0x4d, 0x5c,
+        0xc1, 0x92, 0x9d, 0x91, 0xb6, 0x36, 0x40, 0xfe,
+        0xe8, 0xf5, 0x52, 0xb5, 0x10, 0x8c, 0x4e, 0xa6,
+        0x20, 0xbe, 0xdb, 0x7b, 0xbc, 0xa0, 0x31, 0x64,
+    ]),
+    initdata_magic: Some(0x0c89_c357_8392_04b8),
+}];
+
+/// The same T8122 image for the start experiment (`t8122_start`, only with
+/// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version.
+pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage {
+    name: "T8122 C0 firmware-compat 14.8.3 (start experiment)",
+    uuid: Some(T8122_C0_UUID),
+    stkg_sha256: None,
+    initdata_magic: Some(G15_V14_8_3_INITDATA),
+}];
+
+const IMAGE_INFO_OFFSET: usize = 0x4200;
+/// Size of the identifying part of the image-info header.
+const IMAGE_INFO_SIZE: usize = 0x38;
+/// First word of the header: a branch over it (`b +0x44`).
+const IMAGE_INFO_BRANCH: u32 = 0x1400_0011;
+/// Header magic, "uuid".
+const IMAGE_INFO_MAGIC: u32 = 0x6469_7575;
+
+/// Parsed image-info header: UUID, patchbay offset and size, TEXT size.
+pub(crate) struct ImageInfo {
+    pub(crate) uuid: [u8; 16],
+    pub(crate) patchbay: core::ops::Range<usize>,
+    pub(crate) text_size: usize,
+}
+
+pub(crate) fn image_info(text: &[u8]) -> Option<ImageInfo> {
+    let hdr = text.get(IMAGE_INFO_OFFSET..IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE)?;
+    let word = |i: usize| u32::from_le_bytes([hdr[i], hdr[i + 1], hdr[i + 2], hdr[i + 3]]);
+    if word(0) != IMAGE_INFO_BRANCH || word(4) != IMAGE_INFO_MAGIC || word(8) != 5 {
+        return None;
+    }
+    let mut uuid = [0u8; 16];
+    uuid.copy_from_slice(&hdr[0x14..0x24]);
+    let start = word(0x2c) as usize;
+    let end = start.checked_add(word(0x30) as usize)?;
+    if word(0x34) as usize != text.len() {
+        return None;
+    }
+    text.get(start..end)?;
+    Some(ImageInfo {
+        uuid,
+        patchbay: start..end,
+        text_size: word(0x34) as usize,
+    })
+}
+
+/// The layout of a loaded GPU firmware the runtime accepts: segment sizes and VAs, and the
+/// per-boot words of the text segment that identification ignores.
+#[derive(Debug)]
+pub(crate) struct Layout {
+    pub(crate) text_size: u64,
+    pub(crate) data_size: u64,
+    pub(crate) vas: [u64; 2],
+    /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing.
+    pub(crate) entropy_tag_offset: usize,
+    pub(crate) entropy_tag: [u8; 8],
+    /// An additional independent boot value, when required by this image.
+    extra_boot_value: Option<(usize, [u8; 8])>,
+    /// Expected boot-table location and byte count in the image header.
+    boot_table: Option<(u32, u32)>,
+    /// The firmware version, for logs.
+    pub(crate) version: &'static str,
+}
+
+pub(crate) static T6030_LAYOUT: Layout = Layout {
+    text_size: 0x5c000,
+    data_size: 0x114000,
+    vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000],
+    entropy_tag_offset: 0x58e88,
+    entropy_tag: *b"GKTS\x08\x00\x00\x00",
+    extra_boot_value: None,
+    boot_table: None,
+    version: "RTKit-2419.140.12.release",
+};
+
+/// T8122 firmware-compat 14.8.3 segment sizes, virtual addresses and entropy-tag bounds.
+pub(crate) static T8122_LAYOUT: Layout = Layout {
+    text_size: 0x64000,
+    data_size: 0xb0000,
+    vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000],
+    entropy_tag_offset: 0x5d54a,
+    entropy_tag: *b"GKTS\x08\x00\x00\x00",
+    extra_boot_value: None,
+    boot_table: None,
+    version: "RTKit-2419.140.12.release",
+};
+
+pub(crate) const TEXT_SHA256: [u8; 32] = [
+    0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5];
+
+/// T8122 C0, firmware-compat 26.6.2 segment and loader-mutable bounds.
+pub(crate) static T8122_LAYOUT_26: Layout = Layout {
+    text_size: 393216,
+    data_size: 1228800,
+    vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_0000],
+    entropy_tag_offset: 380492,
+    entropy_tag: *b"GKTS\x08\x00\x00\x00",
+    extra_boot_value: Some((380719, *b"ECAP\x08\x00\x00\x00")),
+    boot_table: Some((380492, 561)),
+    version: "RTKit-3255.160.4.release",
+};
+
+fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool {
+    if text.len() != layout.text_size as usize {
+        return false;
+    }
+    if let Some((offset, count)) = layout.boot_table {
+        let Some(header) = text.get(0x22c..0x234) else {
+            return false;
+        };
+        if header[..4] != offset.to_le_bytes() || header[4..] != count.to_le_bytes() {
+            return false;
+        }
+        let Some(end) = (offset as usize).checked_add(count as usize) else {
+            return false;
+        };
+        if text.get(offset as usize..end).is_none() {
+            return false;
+        }
+    }
+    let fields = [
+        Some((layout.entropy_tag_offset, layout.entropy_tag)),
+        layout.extra_boot_value,
+    ];
+    // Validate every field before touching even the first boot value.
+    for (offset, tag) in fields.into_iter().flatten() {
+        let Some(end) = offset.checked_add(16) else {
+            return false;
+        };
+        let Some(field) = text.get(offset..end) else {
+            return false;
+        };
+        if field[..8] != tag {
+            return false;
+        }
+        if let Some((base, count)) = layout.boot_table {
+            if offset < base as usize || end > base as usize + count as usize {
+                return false;
+            }
+        }
+    }
+    if let Some((other, _)) = layout.extra_boot_value {
+        if other < layout.entropy_tag_offset + 16 && layout.entropy_tag_offset < other + 16 {
+            return false;
+        }
+    }
+    for (offset, _) in fields.into_iter().flatten() {
+        text[offset + 8..offset + 16].fill(0);
+    }
+    true
+}
+
+#[derive(Debug)]
+pub(crate) struct Firmware {
+    pub(crate) resources: agx_resources::Resources,
+    /// The InitData version validated for the identified image.
+    pub(crate) initdata_magic: u64,
+    /// The accepted layout.
+    pub(crate) layout: &'static Layout,
+}
+
+impl Firmware {
+    pub(crate) fn profile(&self) -> super::m3_profile::Profile {
+        if core::ptr::eq(self.layout, &T8122_LAYOUT_26) {
+            super::m3_profile::Profile::Extended
+        } else {
+            super::m3_profile::Profile::Legacy
+        }
+    }
+
+    pub(crate) const fn version(&self) -> &'static str {
+        self.layout.version
+    }
+
+    fn matching_layout(layout: &Layout, resources: &agx_resources::Resources) -> bool {
+        resources.regions[4].size == layout.text_size
+            && resources.regions[5].size == layout.data_size
+            && resources.firmware_vas == layout.vas
+    }
+}
+
+/// Only the admitted extended image has this host-owned boot descriptor slot.
+fn loader_span(profile: super::m3_profile::Profile, data_bytes: u64, input_bytes: usize)
+    -> Option<core::ops::Range<usize>> {
+    if profile != super::m3_profile::Profile::Extended || input_bytes != 53 * 40
+        || data_bytes != T8122_LAYOUT_26.data_size { return None; }
+    let start: usize = 0x12_28e0;
+    let end = start.checked_add(input_bytes)?;
+    if end as u64 > data_bytes { return None; }
+    Some(start..end)
+}
+
+#[cfg(not(test))]
+impl Firmware {
+    /// Call after RTKit application startup and before exposing the InitData root.
+    /// The protocol grants the host this slot only during that boot interval.
+    pub(crate) fn publish_io_table(
+        &self,
+        pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+        descriptors: &[u8],
+    ) -> kernel::error::Result {
+        use kernel::{c_str, io::mem::{Mem, MemFlag}, prelude::*};
+        let data = self.resources.regions[5];
+        let span = loader_span(self.profile(), data.size, descriptors.len()).ok_or(EINVAL)?;
+        let node = pdev.as_ref().of_node().ok_or(ENODEV)?;
+        let resource = agx_resources::reserved_resource(&node, c_str!("fw-data"))?;
+        if resource.start() != data.base || resource.size() != data.size { return Err(EINVAL); }
+        // SAFETY: This exact admitted no-map reservation remains firmware-owned.
+        // Only its bounded host boot slot is accessed, before root publication.
+        let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?;
+        if span.end > mapping.size() { return Err(EINVAL); }
+        // Validate the whole slot before changing any byte. Unexpected prior
+        // contents indicate a protocol/ownership mismatch, not a patchable image.
+        for offset in span.clone() {
+            // SAFETY: span is bounded by the retained mapping above.
+            if unsafe { mapping.ptr().add(offset).read_volatile() } != 0 { return Err(EBUSY); }
+        }
+        for (offset, byte) in span.zip(descriptors.iter().copied()) {
+            // SAFETY: same bounded host slot, exclusive under the boot protocol.
+            unsafe { mapping.ptr().add(offset).write_volatile(byte) };
+        }
+        crate::m3::agx_memory::publish();
+        Ok(())
+    }
+}
+
+#[cfg(not(test))]
+pub(crate) fn identify_loaded(
+    pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+    soc: &'static crate::m3::m3_soc::Soc,
+    resources: agx_resources::Resources,
+    experiment: Option<&crate::m3::t8122_start::Experiment>,
+) -> kernel::error::Result<Firmware> {
+    use kernel::{
+        bindings, c_str,
+        io::mem::{Mem, MemFlag},
+        prelude::*,
+    };
+
+    let extended=crate::m3::t8122_start::is_t8122(soc)
+        && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended;
+    if extended && experiment.is_some() {return Err(EINVAL);}
+    let layout = if extended {&T8122_LAYOUT_26} else {soc.firmware.ok_or(ENODEV)?};
+    if !Firmware::matching_layout(layout, &resources) {
+        dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name);
+        return Err(ENODEV);
+    }
+    let node = pdev.as_ref().of_node().ok_or(ENODEV)?;
+    let text = agx_resources::reserved_resource(&node, c_str!("fw-text"))?;
+    if text.start() != resources.regions[4].base || text.size() != layout.text_size {
+        return Err(EINVAL);
+    }
+    let mapping = unsafe { Mem::try_new(text, MemFlag::WB.into()) }?;
+    let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) };
+    let mut canonical = KVec::new();
+    canonical.extend_from_slice(bytes, GFP_KERNEL)?;
+    if !normalize_boot_entropy(layout, &mut canonical) {
+        return Err(ENODEV);
+    }
+    let mut digest = [0u8; 32];
+    // SAFETY: the initialized private copy and distinct digest live for the
+    // synchronous SHA-256 call. Normalization never writes loaded firmware.
+    unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) };
+    // The T8122 start experiment identifies the same image, with its own InitData version.
+    let images = match experiment {
+        Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..],
+        None if extended => &T8122_IMAGES_26[..],
+        None => soc.images,
+    };
+    let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?;
+    let initdata_magic = match experiment {
+        Some(experiment) => experiment.initdata_version(),
+        None => image.initdata_magic.ok_or(ENODEV)?,
+    };
+    Ok(Firmware { resources, initdata_magic, layout })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use agx_resources::{Region, Resources};
+
+    fn image_header(uuid: [u8; 16]) -> Vec<u8> {
+        let mut text = vec![0; T6030_LAYOUT.text_size as usize];
+        let size = text.len() as u32;
+        for (offset, value) in [
+            (0, IMAGE_INFO_BRANCH), (4, IMAGE_INFO_MAGIC), (8, 5),
+            (0x2c, size - 32), (0x30, 32), (0x34, size),
+        ] {
+            header_word(&mut text, offset, value);
+        }
+        text[IMAGE_INFO_OFFSET + 0x14..IMAGE_INFO_OFFSET + 0x24].copy_from_slice(&uuid);
+        text
+    }
+
+    fn header_word(text: &mut [u8], offset: usize, value: u32) {
+        let start = IMAGE_INFO_OFFSET + offset;
+        text[start..start + 4].copy_from_slice(&value.to_le_bytes());
+    }
+
+    fn admits_text(image: &KnownImage, text: &[u8], digest: &[u8; 32]) -> bool {
+        image.accepts_runtime(image_info(text).map(|info| info.uuid), digest)
+    }
+
+    #[test]
+    fn uuid_admission_requires_header_bounds_and_exact_text_size() {
+        let image = &KNOWN_IMAGES[1];
+        let text = image_header(image.uuid.unwrap());
+        assert!(admits_text(image, &text, &[0; 32]));
+        let size = text.len() as u32;
+        for (offset, value) in [
+            (0x34, 0), (0x34, size - 1), (0x34, size + 1),
+            (0x2c, size), (0x2c, u32::MAX), (0x30, u32::MAX),
+        ] {
+            let mut bad = text.clone();
+            header_word(&mut bad, offset, value);
+            assert!(!admits_text(image, &bad, &[0; 32]), "offset {offset:#x}, {value:#x}");
+        }
+        let info = image_info(&text).unwrap();
+        assert_eq!(info.patchbay, text.len() - 32..text.len());
+        assert_eq!(info.text_size, text.len());
+    }
+
+    #[test]
+    fn uuid_admission_rejects_bad_header_words_and_truncation() {
+        let image = &KNOWN_IMAGES[1];
+        let text = image_header(image.uuid.unwrap());
+        for offset in [0, 4, 8] {
+            let mut bad = text.clone();
+            header_word(&mut bad, offset, 0);
+            assert!(!admits_text(image, &bad, &[0; 32]));
+        }
+        for len in [0, IMAGE_INFO_OFFSET, IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE - 1, text.len() - 1] {
+            assert!(!admits_text(image, &text[..len], &[0; 32]));
+        }
+    }
+
+    #[test]
+    fn runtime_admission_requires_every_recorded_identity() {
+        let image = KnownImage {
+            name: "test", uuid: Some([1; 16]), stkg_sha256: Some([2; 32]),
+            initdata_magic: Some(3),
+        };
+        let text = image_header([1; 16]);
+        assert!(admits_text(&image, &text, &[2; 32]));
+        assert!(!admits_text(&image, &text, &[0; 32]));
+        assert!(!admits_text(&image, &image_header([0; 16]), &[2; 32]));
+        assert!(!image.accepts_runtime(None, &[2; 32]));
+        let anonymous = KnownImage { uuid: None, stkg_sha256: None, ..image };
+        assert!(!admits_text(&anonymous, &text, &[2; 32]));
+    }
+
+    #[test]
+    fn digest_only_admission_preserves_exact_digest_requirement() {
+        let image = &KNOWN_IMAGES[0];
+        assert!(admits_text(image, &[], &TEXT_SHA256));
+        let mut wrong = TEXT_SHA256;
+        wrong[0] ^= 1;
+        assert!(!admits_text(image, &[], &wrong));
+    }
+
+    #[test]
+    fn t8122_admission_requires_qualified_initdata_and_matching_image_identity() {
+        let image = &KNOWN_IMAGES_T8122[0];
+        assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA));
+        let text = image_header(image.uuid.unwrap());
+        assert_eq!(image_info(&text).unwrap().uuid, image.uuid.unwrap());
+        assert!(admits_text(image, &text, &[0; 32]));
+
+        // A qualified version does not admit an unknown or another chip's image.
+        for wrong_uuid in [[0; 16], KNOWN_IMAGES[1].uuid.unwrap()] {
+            assert!(!admits_text(image, &image_header(wrong_uuid), &[0; 32]));
+        }
+        assert!(!image.accepts_runtime(None, &[0; 32]));
+
+        // Matching identity still cannot admit an unqualified InitData layout.
+        let unqualified = KnownImage {
+            name: image.name, uuid: image.uuid, stkg_sha256: image.stkg_sha256,
+            initdata_magic: None,
+        };
+        assert!(!admits_text(&unqualified, &text, &[0; 32]));
+        let unidentified = KnownImage {
+            name: "unknown", uuid: None, stkg_sha256: None,
+            initdata_magic: image.initdata_magic,
+        };
+        assert!(!admits_text(&unidentified, &text, &[0; 32]));
+    }
+
+    #[test]
+    fn t8122_experiment_record_identifies_the_same_image_with_the_g15_version() {
+        let image = &KNOWN_IMAGES_T8122_EXPERIMENT[0];
+        assert_eq!(image.uuid, KNOWN_IMAGES_T8122[0].uuid);
+        assert_eq!(image.stkg_sha256, None);
+        assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA));
+        assert!(KNOWN_IMAGES.iter().all(|i| i.initdata_magic == Some(G15_V14_8_3_INITDATA)));
+        let text = image_header(image.uuid.unwrap());
+        assert!(admits_text(image, &text, &[0; 32]));
+        assert!(!admits_text(image, &image_header(KNOWN_IMAGES[1].uuid.unwrap()), &[0; 32]));
+        assert!(!admits_text(&KNOWN_IMAGES[1], &text, &[0; 32]));
+    }
+
+    #[test]
+    fn each_layout_rejects_the_other_chips_segments() {
+        for (layout, other) in [(&T6030_LAYOUT, &T8122_LAYOUT), (&T8122_LAYOUT, &T6030_LAYOUT)] {
+            let mut regions = [Region { base: 0, size: 0 }; 6];
+            regions[4].size = layout.text_size;
+            regions[5].size = layout.data_size;
+            let mut resources = Resources { regions, firmware_vas: layout.vas };
+            assert!(Firmware::matching_layout(layout, &resources));
+            assert!(!Firmware::matching_layout(other, &resources));
+            resources.firmware_vas[1] += 0x4000;
+            assert!(!Firmware::matching_layout(layout, &resources));
+        }
+    }
+
+    #[test]
+    fn entropy_normalization_changes_only_the_eight_boot_bytes() {
+        for layout in [&T6030_LAYOUT, &T8122_LAYOUT] {
+            let mut text = vec![0xa5; layout.text_size as usize];
+            let tag = layout.entropy_tag_offset;
+            text[tag..tag + 8].copy_from_slice(&layout.entropy_tag);
+            let before = text.clone();
+            assert!(normalize_boot_entropy(layout, &mut text));
+            assert_eq!(&text[..tag + 8], &before[..tag + 8]);
+            assert_eq!(&text[tag + 8..tag + 16], &[0; 8]);
+            assert_eq!(&text[tag + 16..], &before[tag + 16..]);
+        }
+    }
+
+    #[test]
+    fn bad_entropy_bounds_tags_and_lengths_do_not_mutate_text() {
+        for offset in [8, 16, usize::MAX] {
+            let layout = Layout { text_size: 16, entropy_tag_offset: offset, ..T8122_LAYOUT };
+            let mut text = vec![0xa5; 16];
+            assert!(!normalize_boot_entropy(&layout, &mut text));
+            assert_eq!(text, vec![0xa5; 16]);
+        }
+        let layout = Layout { text_size: 16, entropy_tag_offset: 0, ..T8122_LAYOUT };
+        let mut text = vec![0xa5; 16];
+        assert!(!normalize_boot_entropy(&layout, &mut text));
+        assert_eq!(text, vec![0xa5; 16]);
+        text.pop();
+        assert!(!normalize_boot_entropy(&layout, &mut text));
+        assert_eq!(text, vec![0xa5; 15]);
+    }
+
+    fn image_26() -> Vec<u8> {
+        let mut image = vec![0x5a; 393216];
+        image[0x22c..0x230].copy_from_slice(&380492u32.to_le_bytes());
+        image[0x230..0x234].copy_from_slice(&561u32.to_le_bytes());
+        image[380492..380500].copy_from_slice(b"GKTS\x08\x00\x00\x00");
+        image[380719..380727].copy_from_slice(b"ECAP\x08\x00\x00\x00");
+        image
+    }
+
+    #[test]
+    fn newer_image_masks_only_both_boot_values() {
+        let mut image = image_26();
+        let before = image.clone();
+        assert!(normalize_boot_entropy(&T8122_LAYOUT_26, &mut image));
+        for index in 0..image.len() {
+            let expected = if (380500..380508).contains(&index) || (380727..380735).contains(&index)
+            {
+                0
+            } else {
+                before[index]
+            };
+            assert_eq!(image[index], expected, "byte {index}");
+        }
+    }
+
+    #[test]
+    fn invalid_second_field_or_table_does_not_mask_first_value() {
+        for index in [0x22c, 0x230, 380492, 380496, 380719, 380723] {
+            let mut image = image_26();
+            image[index] ^= 1;
+            let before = image.clone();
+            assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image));
+            assert_eq!(image, before);
+        }
+        let mut image = image_26();
+        image.pop();
+        let before = image.clone();
+        assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image));
+        assert_eq!(image, before);
+    }
+
+    #[test]
+    fn extended_record_cannot_replace_legacy_identity() {
+        let newer = &T8122_IMAGES_26[0];
+        let digest = newer.stkg_sha256.unwrap();
+        assert!(newer.accepts_runtime(None, &digest));
+        assert_eq!(newer.initdata_magic, Some(0x0c89_c357_8392_04b8));
+        for index in 0..digest.len() {
+            let mut wrong = digest;
+            wrong[index] ^= 1;
+            assert!(!newer.accepts_runtime(None, &wrong));
+        }
+        assert!(!KNOWN_IMAGES_T8122[0].accepts_runtime(None, &digest));
+        assert!(!newer.accepts_runtime(KNOWN_IMAGES_T8122[0].uuid, &[0; 32]));
+    }
+
+    #[test]
+    fn loader_slot_is_exact_bounded_and_profile_specific() {
+        use super::super::m3_profile::Profile;
+        let length = 53 * 40;
+        assert_eq!(loader_span(Profile::Extended, 0x12c000, length), Some(0x1228e0..0x123128));
+        for size in [0, 0x123127, 0x12bfff, 0x12c001, u64::MAX] {
+            assert!(loader_span(Profile::Extended, size, length).is_none());
+        }
+        for bytes in [0, length - 1, length + 1, usize::MAX] {
+            assert!(loader_span(Profile::Extended, 0x12c000, bytes).is_none());
+        }
+        assert!(loader_span(Profile::Legacy, 0x12c000, length).is_none());
+    }
+
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs
index 44671ee3cd0ab..e0aaf2584284a 100644
--- a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs
@@ -3,6 +3,8 @@ use super::{agx_render::Geometry, agx_render_state::{Program,DepthStencil},
     m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region},
     m3_queue_layout::FirmwareVa, m3_render_sequence::fragment};
 pub(crate) const SIZE:usize=0xc73;
+pub(crate) const EXTENDED_SIZE:usize=0xca0;
+pub(crate) const EXTENDED_FLUSH:usize=0xbdc;
 pub(crate) const REGISTERS:usize=fragment::REGISTERS;
 /// The registers G15G does not take; the others keep the G15S order.
 pub(crate) const G15G_OMIT:[u32;4]=[0x160a8,0x160b8,0xa5a9,0xd429];
@@ -74,16 +76,30 @@ impl Command {
     /// Check references before touching output. The caller owns validation of
     /// client resource mappings; this layer serializes their semantic state.
     pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> {
-        if out.len()<SIZE {return Err(Error::Size);}
+        self.encode_format(out,None)
+    }
+    pub(crate) fn encode_extended(&self, dispatch:super::m3_tiler_command::Dispatch26,
+        out:&mut[u8])->Result<(),Error> {
+        self.encode_format(out,Some(dispatch))
+    }
+    fn encode_format(&self,out:&mut[u8],dispatch:Option<super::m3_tiler_command::Dispatch26>)->Result<(),Error> {
+        let extended=dispatch.is_some();
+        let size=if extended {EXTENDED_SIZE} else {SIZE};
+        if out.len()<size {return Err(Error::Size);}
+        if let Some(native)=&dispatch {
+            if native.incarnation==0 || native.serial==0 || native.usc_window&0xffff_ffff!=0
+                || native.usc_window>=1<<42 || self.register_set!=RegisterSet::G15G {return Err(Error::Address);}
+        }
         if self.context>=64 {return Err(Error::Address);}
-        let seq=self.sequence.at(0,fragment::LENGTH)?;
+        let sequence_size=if extended {0x370} else {fragment::LENGTH};
+        let seq=self.sequence.at(0,sequence_size)?;
         let alias=self.gpu_alias.get();
-        GpuVa::new(alias.checked_add((SIZE-3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?;
+        GpuVa::new(alias.checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?;
         let registers=alias+REGISTERS as u64;
         let s=&self.state;let g=&s.geometry;
         let layer_metadata=self.heap.get().checked_add(if g.layers>1 {0x200}else{0}).ok_or(Error::Address)?;
         GpuVa::new(layer_metadata).map_err(|_|Error::Address)?;
-        let user=super::agx_render_state::compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)
+        let user=super::agx_render_state::compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)
             .map_err(|_|Error::Address)? & !15;
         let width=(g.pixels&0xffff)+1;let height=(g.pixels>>16)+1;
         let tx=width.div_ceil(32);let ty=height.div_ceil(32);
@@ -116,43 +132,85 @@ impl Command {
             (0x101c1,0),(0xd469,0),(0x1a0f9,8),(0x1c838,0),(0x1ca28,user),
             (0xa5a9,0x6000400020),(0xd429,0x200000001),
         ];
-        let bytes=&mut out[..SIZE];bytes.fill(0);
+        let bytes=&mut out[..size];bytes.fill(0);
         word(bytes,0,1); // fragment opcode; work counter remains zero
         word(bytes,0xc,self.context);
         for (offset,value) in [(0x14,seq),(0x20,self.notifier.get()),(0x28,self.manager.get()),
             (0x30,self.scene.get()),(0x38,self.empty.get()),(0x40,self.tilemap.get()),(0x780,registers)] {pointer(bytes,offset,value);}
-        word(bytes,SEQUENCE_LENGTH,fragment::LENGTH as u32);
-        // The qualified patch path updates register MSAA but retains these
-        // producer header values. Representation cleanup must not change that.
-        pointer(bytes,0x48,0x88);word(bytes,0x50,1);
-        bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes());
-        bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes());
-        word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]);
-        pointer(bytes,0x78,u64::from(g.tiles));
+        word(bytes,SEQUENCE_LENGTH,sequence_size as u32);
+        if extended {
+            pointer(bytes,4,dispatch.as_ref().map_or(0,|native|native.serial));
+            word(bytes,0x50,1);
+            word(bytes,0x54,g.isp_blocks);
+            word(bytes,0x60,g.x_blocks);
+            word(bytes,0x64,g.y_blocks);
+            word(bytes,0x68,g.blocks);
+            word(bytes,0x78,g.tiles);
+        } else {
+            pointer(bytes,0x48,0x88);word(bytes,0x50,1);
+            bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes());
+            bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes());
+            word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]);
+            pointer(bytes,0x78,u64::from(g.tiles));
+        }
         let g15g=self.register_set==RegisterSet::G15G;
         let mut count=0;
-        for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) {
-            let n=count;count+=1;
-            word(bytes,REGISTERS+n*REGISTER_STRIDE,register);pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value);
+        let mut emit=|register,value| {
+            word(bytes,REGISTERS+count*REGISTER_STRIDE,register);
+            pointer(bytes,REGISTERS+count*REGISTER_STRIDE+4,value);
+            count+=1;
+        };
+        for (register,mut value) in values {
+            if extended {
+                if matches!(register,0x160b8|0x120a1|0x1c838|0x1ca28) {continue;}
+                if register==0x10791 {emit(0xa301,0);}
+                if register==0x16068 {value=(value&!0x3ffff)|0x3717f;}
+                if matches!(register,0x15109|0x15101) {value&=!3;}
+            } else if g15g && G15G_OMIT.contains(&register) {continue;}
+            emit(register,value);
+        }
+        if let Some(native)=&dispatch {
+            for register in [0x160e0,0x1499,0xa341] {emit(register,fragment::UUID.into());}
+            emit(0x1c838,0);
+            emit(0x1ca28,user);
+            emit(0x10069,native.usc_window);
         }
         bytes[0x788..0x78a].copy_from_slice(&(count as u16).to_le_bytes());
         bytes[0x78a..0x78c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes());
         // Start3DJobParameters3: mirror records use their actual packed widths.
         for (offset,value) in [(0,s.depth_bias),(0x10,s.scissor),(0x20,s.query),
-            (0x168,s.partial_background.resources),(0x170,s.partial_background.address),
+            (0x168,if extended {s.background.resources}else{s.partial_background.resources}),
+            (0x170,if extended {s.background.address}else{s.partial_background.address}),
             (0x198,s.partial_background.resources),(0x1a0,s.partial_background.address),
             (0x1a8,s.zls_control),(0x1b8,s.depth.base),(0x1c0,s.depth.stride),
             (0x1c8,s.depth_compression.stride),(0x1e0,s.depth_compression.base),
             (0x1d0,s.depth.base),(0x1d8,s.depth.base),
             (0x1e8,s.stencil.base),(0x1f0,s.stencil.stride),(0x200,s.stencil.base),(0x208,s.stencil.base),
             (0x1f8,s.stencil_compression.stride),(0x210,s.stencil_compression.base),
-            (0x26c,s.partial_eot.address),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] {
-            pointer(bytes,PARAMETERS+offset,value);
+            (0x26c,if extended {s.eot.address}else{s.partial_eot.address}),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] {
+            pointer(bytes,PARAMETERS+offset+if extended && offset<=0x20 {0x10}else{0},value);
         }
         for (offset,value) in [(0x1b0,0x04040404),(0x228,s.mirror_tilebuffer_control),
             (0x230,s.isp_control),(0x238,width),(0x23c,height),(0x240,0x100000),
-            (0x248,s.mirror_tile_mode),(0x264,s.partial_eot.resources as u32),(0x284,s.partial_eot.resources as u32),
+            (0x248,s.mirror_tile_mode),(0x264,if extended {s.eot.resources as u32}else{s.partial_eot.resources as u32}),(0x284,s.partial_eot.resources as u32),
             (0x298,s.depth_clear),(0x29c,s.stencil_clear),(0x2a0,s.sample_size)] {word(bytes,PARAMETERS+offset,value);}
+        if let Some(native)=dispatch {
+            pointer(bytes,0xbc0,self.stamp.get());
+            pointer(bytes,0xbc8,self.fw_stamp.get());
+            word(bytes,0xbd0,self.stamp_value);
+            word(bytes,0xbd4,1);
+            word(bytes,EXTENDED_FLUSH,1);
+            word(bytes,0xbe0,fragment::UUID);
+            pointer(bytes,0xc10,self.timestamps[0].get());
+            pointer(bytes,0xc18,self.timestamps[1].get());
+            pointer(bytes,0xc20,self.user_timestamps[0].get());
+            pointer(bytes,0xc28,self.user_timestamps[1].get());
+            pointer(bytes,0xc46,self.pool.get());
+            bytes[0xc4e]=1;
+            pointer(bytes,0xc57,native.firmware_aux.get());
+            bytes[0xc5f]=native.incarnation;
+            bytes[0xc84]=1;
+        } else {
         word(bytes,0xb88,0x67); // EncoderParams.encoder_id
         word(bytes,0xb90,u32::MAX); // EncoderParams.unk_mask
         word(bytes,0xb94,u32::from(s.process_empty_tiles));
@@ -167,6 +225,46 @@ impl Command {
             pointer(bytes,fragment::COMMAND_TAIL+offset,value);
         }
         bytes[fragment::COMMAND_TAIL+0x36]=1;bytes[fragment::COMMAND_TAIL+0x4f]=1;
+        }
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    #[test]
+    fn native_fragment_uses_distinct_program_mirrors_and_completion_tail() {
+        let fw=|n:u64|FirmwareVa::new(0xffff_fc25_3000_0000+n*0x4000).unwrap();
+        let gpu=|n:u64|GpuVa::new(0x10_0300_0000+n*0x4000).unwrap();
+        let program=|n:u64|Program{address:0x1_0000_0000+n*0x1000,resources:n};
+        let surface=DepthStencil{base:0,stride:0};
+        let command=Command {context:3,register_set:RegisterSet::G15G,
+            sequence:Region::new(fw(0).get(),0x1000).unwrap(),notifier:fw(1),manager:fw(2),scene:fw(3),empty:fw(4),
+            gpu_alias:GpuVa::new(0x72_1000_4000).unwrap(),tilemap:gpu(0),heap:gpu(1),auxiliary:gpu(2),scene_user:gpu(3),
+            scratch:gpu(4),pool:fw(5),stamp:fw(6),fw_stamp:fw(7),stamp_value:0x3d000100,
+            timestamps:[fw(8),fw(9)],user_timestamps:[fw(10),fw(11)],
+            state:State {geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32,
+                super::super::agx_render::Utile::Pixels32,1).unwrap(),multisample:0,merge_upper:[0;2],
+                eot:program(1),background:program(2),partial_eot:program(3),partial_background:program(4),
+                scissor:0,depth_bias:0,query:0,isp_control:0xc000,tilebuffer_control:1,mirror_tilebuffer_control:1,
+                tile_mode:0x280,mirror_tile_mode:0x280,depth:surface,stencil:surface,depth_compression:surface,
+                stencil_compression:surface,zls_control:0,depth_dimensions:0,depth_clear:0,stencil_clear:0,
+                sample_size:4,process_empty_tiles:false}};
+        let dispatch=||super::super::m3_tiler_command::Dispatch26 {incarnation:9,serial:1,peer_stamp:0,
+            usc_window:1<<32,firmware_aux:fw(12),scene_scratch:gpu(4)};
+        let mut bytes=[0xa5;EXTENDED_SIZE+8];
+        command.encode_extended(dispatch(),&mut bytes).unwrap();
+        let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap());
+        let pointer=|at|u64::from_le_bytes(bytes[at..at+8].try_into().unwrap());
+        assert_eq!(word(0xbd0),command.stamp_value);
+        assert_eq!(word(EXTENDED_FLUSH),1);
+        assert_eq!(pointer(0x900),command.state.background.address);
+        assert_eq!(pointer(0x930),command.state.partial_background.address);
+        assert_eq!(bytes[0xc5f],9);
+        assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]);
+        let mut short=[0xa5;EXTENDED_SIZE-1];
+        assert!(command.encode_extended(dispatch(),&mut short).is_err());
+        assert!(short.iter().all(|b|*b==0xa5));
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_init_26.rs b/drivers/gpu/drm/asahi/m3/m3_init_26.rs
new file mode 100644
index 0000000000000..a86432b2bbe2b
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_init_26.rs
@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Allocation and pointer records for the extended M3 initialization format.
+//! Owners supply mapped regions; these builders never allocate or publish DMA.
+
+use super::m3_init_layout::{Error, Region, Root};
+use super::m3_init_storage::{self as storage, Allocation, Initial};
+
+pub(crate) const COUNT: usize = 56;
+pub(crate) const DYNAMIC_STATE: usize = 49;
+pub(crate) const PERFORMANCE_CONTROL: usize = 50;
+pub(crate) const RECOVERY_ACTIVITY: usize = 51;
+pub(crate) const SENSOR_VALUE: usize = 52;
+pub(crate) const RETENTION_STATE: usize = 53;
+pub(crate) const GLOBAL_POOL: usize = 54;
+pub(crate) const GLOBAL_PARAMETERS: usize = 55;
+pub(crate) const RUNTIME_SIZE: usize = 0x4b8;
+pub(crate) const LOG_THREADS: usize = 9;
+pub(crate) const RX_BYTES: usize = 0x48;
+pub(crate) const LOG_PAYLOAD_BYTES: usize = 0xd8;
+
+/// Extend the existing M3 owner indices instead of introducing another owner graph.
+pub(crate) fn allocation(index: usize) -> Result<Allocation, Error> {
+    let size = match index {
+        0..=23 if index % 2 == 0 => 0x30,
+        0..=23 => 0x1800,
+        24 | 26 | 30 | 32 => 0x30,
+        25 => 0x4000,
+        27 | 33 => 0x4800,
+        28 => LOG_THREADS * 0x30,
+        29 => LOG_THREADS * 256 * RX_BYTES,
+        31 => 512 * RX_BYTES,
+        34 => 0x40,
+        35 => 0x3fc0,
+        36 => 0xc8,
+        37 => 0x4000,
+        38 => RUNTIME_SIZE,
+        39 => 0x2658,
+        40 => 0x88,
+        41 => LOG_THREADS * 256 * LOG_PAYLOAD_BYTES,
+        42 => 0xc18,
+        43 => 0x1248,
+        44 => 0xe10,
+        45 => 0xe48,
+        46 => 0x670,
+        47 => 0xeaf0,
+        48 => 0x20,
+        DYNAMIC_STATE | RETENTION_STATE | GLOBAL_POOL | GLOBAL_PARAMETERS => 0x4000,
+        PERFORMANCE_CONTROL => 0x6578,
+        RECOVERY_ACTIVITY => 0x60,
+        SENSOR_VALUE => 4,
+        _ => return Err(Error::Bounds),
+    };
+    // Separate page-aligned owners, with room for the largest receive allocation.
+    // This namespace is disjoint from the existing fixed work and timestamp arenas.
+    Ok(Allocation {
+        address: 0xffff_fc21_0000_0000 + (index as u64) * 0x10_0000,
+        size,
+        gpu_shared: matches!(index, GLOBAL_POOL | GLOBAL_PARAMETERS),
+        initial: Initial::Zero,
+    })
+}
+
+pub(crate) fn cached(index: usize) -> bool {
+    matches!(index, 39 | PERFORMANCE_CONTROL)
+}
+
+pub(crate) struct PointerGraph<'a> {
+    pub(crate) owners: &'a [Region; COUNT],
+    pub(crate) scratch: Region,
+    pub(crate) pool_alias: u64,
+    pub(crate) parameter_alias: u64,
+}
+
+impl PointerGraph<'_> {
+    /// Check the complete graph, including padded mapping ownership, before encoding it.
+    fn addresses(&self) -> Result<[u64; COUNT], Error> {
+        let mut addresses = [0; COUNT];
+        for (index, owner) in self.owners.iter().enumerate() {
+            let size = allocation(index)?.size;
+            addresses[index] = owner.at(0, size)?;
+            if addresses[index] & 0x3fff != 0
+                || addresses[index] < 0xffff_fc20_0000_0000
+                || addresses[index] >= 0xffff_fc2f_fc00_0000
+            {
+                return Err(Error::Address);
+            }
+            let mapped = (size + 0x3fff) & !0x3fff;
+            let end = addresses[index]
+                .checked_add(mapped as u64)
+                .ok_or(Error::Address)?;
+            if end > 0xffff_fc2f_fc00_0000 {
+                return Err(Error::Address);
+            }
+            for prior in 0..index {
+                let prior_size = (allocation(prior)?.size + 0x3fff) & !0x3fff;
+                if addresses[index] < addresses[prior] + prior_size as u64
+                    && addresses[prior] < addresses[index] + mapped as u64
+                {
+                    return Err(Error::Address);
+                }
+            }
+        }
+        let scratch = self.scratch.at(0, 0x8000)?;
+        let scratch_end = scratch.checked_add(0x8000).ok_or(Error::Address)?;
+        if scratch & 0x3fff != 0
+            || scratch < 0xffff_fc20_0000_0000
+            || scratch_end > 0xffff_fc2f_fc00_0000
+        {
+            return Err(Error::Address);
+        }
+        for (index, address) in addresses.iter().enumerate() {
+            let bytes = (allocation(index)?.size + 0x3fff) & !0x3fff;
+            if scratch < *address + bytes as u64 && *address < scratch_end {
+                return Err(Error::Address);
+            }
+        }
+        for alias in [self.pool_alias, self.parameter_alias] {
+            if alias & 0x3fff != 0 || !(0x70_0000_0000..0x70_0200_0000).contains(&alias) {
+                return Err(Error::Address);
+            }
+        }
+        if self.pool_alias == self.parameter_alias {
+            return Err(Error::Address);
+        }
+        Ok(addresses)
+    }
+
+    pub(crate) fn root(&self, output: &mut [u8]) -> Result<(), Error> {
+        self.addresses()?;
+        Root {
+            region_a: self.owners[storage::REGION_A],
+            runtime: self.owners[storage::RUNTIME_POINTERS],
+            globals: self.owners[storage::GLOBALS],
+            new_a: self.owners[storage::RUNTIME_FLAGS],
+            control: self.owners[storage::CONTROL_REGION],
+            power: self.owners[storage::GLOBALS_POWER],
+        }
+        .encode_26(output, self.owners[DYNAMIC_STATE])
+    }
+
+    pub(crate) fn runtime(&self, output: &mut [u8]) -> Result<(), Error> {
+        let addresses = self.addresses()?;
+        let scratch = self.scratch.at(0, 0x8000)?;
+        let bytes = output.get_mut(..RUNTIME_SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        let pointer = |bytes: &mut [u8], offset: usize, value: u64| {
+            bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
+        };
+        pointer(bytes, 0x18, scratch);
+        pointer(bytes, 0x2d0, self.parameter_alias);
+        pointer(bytes, 0x2e0, self.pool_alias);
+        bytes[0x250..0x254].copy_from_slice(&1u32.to_le_bytes());
+        bytes[0x2f8..0x2fc].copy_from_slice(&4u32.to_le_bytes());
+        for (offset, owner) in [
+            (0, storage::HARDWARE_DATA),
+            (0x10, storage::UNKNOWN_PAIR),
+            (0x1c0, 26),
+            (0x1c8, 27),
+            (0x1d0, 28),
+            (0x1d8, 29),
+            (0x1e0, 30),
+            (0x1e8, 31),
+            (0x1f0, 32),
+            (0x1f8, 33),
+            (0x200, storage::FWLOG_PAYLOAD),
+            (0x254, storage::UNKNOWN_C0),
+            (0x25c, storage::UNKNOWN_C1),
+            (0x264, storage::UNKNOWN_C3),
+            (0x26c, RECOVERY_ACTIVITY),
+            (0x2d8, GLOBAL_PARAMETERS),
+            (0x2e8, GLOBAL_POOL),
+            (0x469, PERFORMANCE_CONTROL),
+        ] {
+            pointer(bytes, offset, addresses[owner]);
+        }
+        for channel in 0..13 {
+            let offset = 0x20 + channel * 0x20;
+            let state = addresses[channel * 2];
+            pointer(bytes, offset, state);
+            pointer(bytes, offset + 8, state + 0x10);
+            pointer(bytes, offset + 16, state + 0x20);
+            pointer(bytes, offset + 24, addresses[channel * 2 + 1]);
+        }
+        Ok(())
+    }
+
+    pub(crate) fn firmware_control(&self, output: &mut [u8]) -> Result<(), Error> {
+        let addresses = self.addresses()?;
+        let bytes = output
+            .get_mut(..allocation(storage::CONTROL_REGION)?.size)
+            .ok_or(Error::Size)?;
+        bytes.fill(0);
+        bytes[0x4f88..0x4f90].copy_from_slice(&addresses[storage::FW_CONTROL_STATE].to_le_bytes());
+        bytes[0x4f90..0x4f98].copy_from_slice(&addresses[storage::FW_CONTROL_RING].to_le_bytes());
+        bytes[0x4fd4..0x4fd8].copy_from_slice(&1u32.to_le_bytes());
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn owners(delta: u64) -> [Region; COUNT] {
+        core::array::from_fn(|index| {
+            let item = allocation(index).unwrap();
+            Region::new(item.address + delta, item.size).unwrap()
+        })
+    }
+
+    fn graph(owners: &[Region; COUNT]) -> PointerGraph<'_> {
+        PointerGraph {
+            owners,
+            scratch: Region::new(0xffff_fc22_0000_0000, 0x8000).unwrap(),
+            pool_alias: 0x70_0000_0000,
+            parameter_alias: 0x70_0000_8000,
+        }
+    }
+
+    #[test]
+    fn complete_plan_has_disjoint_page_owners_and_nine_log_threads() {
+        let owners = owners(0);
+        assert!(graph(&owners).addresses().is_ok());
+        assert_eq!(allocation(28).unwrap().size, 0x1b0);
+        assert_eq!(allocation(29).unwrap().size, 0x28800);
+        assert_eq!(allocation(41).unwrap().size, 0x79800);
+        assert!(allocation(COUNT).is_err());
+    }
+
+    #[test]
+    fn runtime_references_relocate_and_keep_the_packed_power_pointer() {
+        for delta in [0, 0x1000_0000] {
+            let owners = owners(delta);
+            let mut output = [0xa5; RUNTIME_SIZE + 8];
+            graph(&owners).runtime(&mut output).unwrap();
+            let read = |at| u64::from_le_bytes(output[at..at + 8].try_into().unwrap());
+            assert_eq!(
+                read(0x469),
+                allocation(PERFORMANCE_CONTROL).unwrap().address + delta
+            );
+            assert_eq!(read(0x1a0), allocation(24).unwrap().address + delta);
+            assert_eq!(read(0x1b8), allocation(25).unwrap().address + delta);
+            assert_eq!(read(0x2d0), 0x70_0000_8000);
+            assert_eq!(&output[RUNTIME_SIZE..], &[0xa5; 8]);
+        }
+    }
+
+    #[test]
+    fn every_short_runtime_rejects_before_mutation() {
+        let owners = owners(0);
+        for length in 0..RUNTIME_SIZE {
+            let mut output = [0xa5; RUNTIME_SIZE];
+            assert_eq!(
+                graph(&owners).runtime(&mut output[..length]),
+                Err(Error::Size)
+            );
+            assert_eq!(output, [0xa5; RUNTIME_SIZE]);
+        }
+    }
+
+    #[test]
+    fn every_owner_is_checked_before_any_record_is_written() {
+        for index in 0..COUNT {
+            let mut owners = owners(0);
+            let item = allocation(index).unwrap();
+            owners[index] = Region::new(item.address, item.size - 1).unwrap();
+            let mut output = [0xa5; RUNTIME_SIZE];
+            assert!(graph(&owners).runtime(&mut output).is_err());
+            assert_eq!(output, [0xa5; RUNTIME_SIZE]);
+        }
+        let mut owners = owners(0);
+        owners[1] = owners[0];
+        let mut output = [0xa5; RUNTIME_SIZE];
+        assert!(graph(&owners).runtime(&mut output).is_err());
+        assert_eq!(output, [0xa5; RUNTIME_SIZE]);
+    }
+    #[test]
+    fn only_atomic_controller_owners_are_firmware_cacheable() {
+        let selected: Vec<_> = (0..COUNT).filter(|index| cached(*index)).collect();
+        assert_eq!(selected, vec![39, PERFORMANCE_CONTROL]);
+    }
+
+    #[test]
+    fn scratch_requires_its_firmware_host_window() {
+        let owners = owners(0);
+        let mut value = graph(&owners);
+        value.scratch = Region::new(0xffff_fc10_0000_0000, 0x8000).unwrap();
+        let mut output = [0xa5; RUNTIME_SIZE];
+        assert_eq!(value.runtime(&mut output), Err(Error::Address));
+        assert_eq!(output, [0xa5; RUNTIME_SIZE]);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs
index 92cc818e030cc..75f06abe6f411 100644
--- a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs
@@ -78,13 +78,38 @@ impl Root {
             (root::CONTROL, self.control.at(0, CONTROL_SIZE)?),
             (root::POWER, self.power.at(0, 2292)?),
         ];
-        let bytes = out.get_mut(..ROOT_SIZE).ok_or(Error::Size)?;
+        Self::write_root(out, ROOT_SIZE, 0x0c08_e21e_8380_0490, &references)
+    }
+
+    /// Encode the extended root without admitting a firmware image or publishing it.
+    pub(crate) fn encode_26(&self, out: &mut [u8], extra: Region) -> Result<(), Error> {
+        let references = [
+            (root::REGION_A, self.region_a.at(0, 0x4000)?),
+            (root::RUNTIME, self.runtime.at(0, 0x4b8)?),
+            (root::GLOBALS, self.globals.at(0, 0xe48)?),
+            (root::NEW_A, self.new_a.at(0, 0x20)?),
+            (root::CONTROL, self.control.at(0, 0xeaf0)?),
+            (root::POWER, self.power.at(0, 0x670)?),
+            (0xc0, extra.at(0, 0x4000)?),
+        ];
+        if references.iter().any(|(_, address)| address & 7 != 0) {
+            return Err(Error::Address);
+        }
+        Self::write_root(out, 0xc8, 0x0c89_c357_8392_04b8, &references)
+    }
+
+    fn write_root(
+        out: &mut [u8],
+        size: usize,
+        version: u64,
+        references: &[(usize, u64)],
+    ) -> Result<(), Error> {
+        let bytes = out.get_mut(..size).ok_or(Error::Size)?;
         bytes.fill(0);
-        // Opaque qualified G15/V14_8_3 version tuple, not a semantic version.
-        for (i, value) in [0x490u16, 0x8380, 0xe21e, 0xc08].iter().enumerate() {
-            bytes[root::VERSION + i * 2..root::VERSION + i * 2 + 2].copy_from_slice(&value.to_le_bytes());
+        pointer(bytes, root::VERSION, version);
+        for &(offset, value) in references {
+            pointer(bytes, offset, value);
         }
-        for (offset, value) in references { pointer(bytes, offset, value); }
         word(bytes, root::HOST_ALLOCATIONS, 1);
         bytes[root::PAGE_SIZE..root::PAGE_SIZE + 2].copy_from_slice(&0x4000u16.to_le_bytes());
         bytes[root::PAGE_BITS] = 14;
@@ -193,3 +218,107 @@ impl ControlRegion {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod root_format_tests {
+    use super::*;
+
+    fn allocations() -> (Root, Region) {
+        let region =
+            |index: u64| Region::new(0xffff_fc01_0000_0000 + index * 0x10000, 0x10000).unwrap();
+        (
+            Root {
+                region_a: region(1),
+                runtime: region(2),
+                globals: region(3),
+                new_a: region(4),
+                control: region(5),
+                power: region(6),
+            },
+            region(7),
+        )
+    }
+
+    #[test]
+    fn root_revision_changes_only_the_version_and_extension() {
+        let (root, extra) = allocations();
+        let mut older = [0x55; 0xd0];
+        let mut newer = older;
+        root.encode(&mut older).unwrap();
+        root.encode_26(&mut newer, extra).unwrap();
+        assert_eq!(&older[8..0xc0], &newer[8..0xc0]);
+        assert_eq!(
+            u64::from_le_bytes(newer[..8].try_into().unwrap()),
+            0x0c89_c357_8392_04b8
+        );
+        assert_eq!(
+            u64::from_le_bytes(newer[0xc0..0xc8].try_into().unwrap()),
+            0xffff_fc01_0007_0000
+        );
+        assert_eq!(&older[0xc0..], &[0x55; 16]);
+        assert_eq!(&newer[0xc8..], &[0x55; 8]);
+        for (offset, index) in [
+            (8, 1),
+            (0x18, 2),
+            (0x20, 3),
+            (0xa8, 4),
+            (0xb0, 5),
+            (0xb8, 6),
+        ] {
+            assert_eq!(
+                u64::from_le_bytes(newer[offset..offset + 8].try_into().unwrap()),
+                0xffff_fc01_0000_0000 + index * 0x10000
+            );
+        }
+    }
+
+    #[test]
+    fn every_short_destination_is_unchanged() {
+        let (root, extra) = allocations();
+        for length in 0..0xc8 {
+            let mut output = [0xa5; 0xc8];
+            assert_eq!(
+                root.encode_26(&mut output[..length], extra),
+                Err(Error::Size)
+            );
+            assert_eq!(output, [0xa5; 0xc8]);
+        }
+    }
+
+    #[test]
+    fn every_reference_requires_its_own_complete_allocation() {
+        for (index, required) in [0x4000, 0x4b8, 0xe48, 0x20, 0xeaf0, 0x670, 0x4000]
+            .into_iter()
+            .enumerate()
+        {
+            for (size, aligned, valid) in [
+                (required - 1, true, false),
+                (required, false, false),
+                (required, true, true),
+            ] {
+                let (mut root, mut extra) = allocations();
+                let reference = match index {
+                    0 => &mut root.region_a,
+                    1 => &mut root.runtime,
+                    2 => &mut root.globals,
+                    3 => &mut root.new_a,
+                    4 => &mut root.control,
+                    5 => &mut root.power,
+                    _ => &mut extra,
+                };
+                *reference =
+                    Region::new(0xffff_fc02_0000_0000 + u64::from(!aligned), size).unwrap();
+                let mut output = [0xa5; 0xc8];
+                let result = root.encode_26(&mut output, extra);
+                assert_eq!(
+                    result.is_ok(),
+                    valid,
+                    "reference {index}, size {size}, aligned {aligned}"
+                );
+                if !valid {
+                    assert_eq!(output, [0xa5; 0xc8]);
+                }
+            }
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs
new file mode 100644
index 0000000000000..6ba456e382e57
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Owned firmware/engine buffers with the qualified M3 page offsets.
+use kernel::prelude::*;
+use crate::{driver, gem, mmu, pgtable::prot};
+
+enum Backing {
+    Coherent(kernel::dma::Coherent<[u8]>),
+    Paged(gem::ObjectRef),
+}
+
+/// Invalidates context 0's non-global translations when dropped. A job's command aliases live in
+/// the context-0 lower tree, which has no bind slot on the adopted UAT (7.1.12 gave it the fixed
+/// slot 0), so unmapping them invalidates nothing by itself. Declared first in a constructor and
+/// last in the job, it drops after every alias.
+pub(crate) struct LowerTlbi;
+
+impl Drop for LowerTlbi {
+    fn drop(&mut self) {
+        crate::mem::sync();
+        crate::mem::tlbi_asid(0);
+        crate::mem::sync();
+    }
+}
+
+pub(crate) struct Buffer {
+    mapping: mmu::KernelMapping,
+    gpu_mapping: Option<mmu::KernelMapping>,
+    // Every client view must preserve the owner's cache and access policy.
+    // A VM switch changes the address space, not the backing's attributes.
+    gpu_protection: prot::Prot,
+    object: Backing,
+    offset: usize,
+    size: usize,
+    _coherent_allocation: Option<crate::m3::agx_memory_stats::Allocation>,
+}
+
+impl Drop for Buffer {
+    fn drop(&mut self) {
+        // A GPU view may be a command alias in the context-0 lower tree, which has no bind slot
+        // to invalidate on unmap: drop it and invalidate context 0 before the backing below is
+        // freed (7.1.12 invalidated inside the unmap).
+        if self.gpu_mapping.take().is_some() {
+            crate::mem::sync();
+            crate::mem::tlbi_asid(0);
+            crate::mem::sync();
+        }
+    }
+}
+
+impl Buffer {
+    pub(crate) fn at(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm,
+        gpu: Option<(&mmu::Vm,u64)>, address: Option<u64>, size: usize,
+        gpu_shared: bool) -> Result<Self> {
+        Self::at_prot(dev,fw,gpu,address,size,
+            if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW},
+            prot::PROT_GPU_SHARED_RW)
+    }
+    pub(crate) fn at_prot(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm,
+        gpu: Option<(&mmu::Vm,u64)>, address: Option<u64>, size: usize,
+        protection: prot::Prot, gpu_protection: prot::Prot) -> Result<Self> {
+        let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize;
+        if size==0 || gpu.is_some_and(|(_,a)| a&0x3fff!=offset as u64) {return Err(EINVAL);}
+        let allocated=(size.checked_add(offset).and_then(|n|n.checked_add(0x3fff)).ok_or(EOVERFLOW)?)&!0x3fff;
+        // The DMA core tries the default CMA area first and falls back to the
+        // page allocator. The graphs outgrow that area, so a CMA miss is the
+        // normal case here and is not reported; only a failed fallback is.
+        // SAFETY: M3 runtime owns the bound platform device for the entire
+        // diagnostic; all mappings retire before normal DMA allocation Drop.
+        let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL,
+            kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?;
+        let pa:usize=object.dma_address().try_into()?;
+        let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?}
+            else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?};
+        let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None};
+        Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size,
+            _coherent_allocation:Some(crate::m3::agx_memory_stats::Allocation::coherent(allocated)?)})
+    }
+    /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the
+    /// old allocation mapped until the replacement has been fully allocated.
+    pub(crate) fn tiler(dev:&driver::AsahiDrmDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result<Self> {
+        if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);}
+        // Large render targets can require 128 MiB of tiler storage. Engines
+        // and firmware address it through UAT, so physically contiguous DMA
+        // backing is unnecessary and can exceed the entire CMA reservation.
+        // GEM provides zeroed pages and the same WC CPU/device coherency used
+        // by M4. Keep both mappings ahead of their backing in drop order.
+        let mut object=gem::new_kernel_object_wc(dev,size)?;
+        let mapping=object.map_into_range(fw,0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,
+            mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?;
+        // Growth retains the cached GPU-only policy of the original TPC and
+        // tilemap allocations; the firmware alias remains shared/uncached.
+        let gpu_protection=prot::Prot::from_pte(crate::m3::m3_pass_layout::Access::GpuCachedRw.pte());
+        let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000,
+            mmu::UAT_PGSZ as u64,gpu_protection,true)?);
+        Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None})
+    }
+    pub(crate) fn va(&self)->u64 {self.mapping.iova()+self.offset as u64}
+    pub(crate) fn gpu_va(&self)->Result<u64> {Ok(self.gpu_mapping.as_ref().ok_or(EINVAL)?.iova()+self.offset as u64)}
+    pub(crate) fn size(&self)->usize {self.size}
+    pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result<mmu::KernelMapping> {
+        let address=self.gpu_va()?&!0x3fff;
+        match &mut self.object {
+            Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_address().try_into()?,object.size(),self.gpu_protection),
+            Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true),
+        }
+    }
+    /// The backing retains its CPU mapping until Buffer is dropped. Callers
+    /// check the range and do not let this pointer escape the current access.
+    fn cpu_ptr(&mut self)->Result<*mut u8> {
+        match &mut self.object {
+            Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::<u8>()),
+            Backing::Paged(object)=>{
+                use kernel::io::IoBase;
+                Ok(object.vmap()?.as_view().as_ptr().cast::<u8>())
+            }
+        }
+    }
+
+    pub(crate) fn swap_gpu_view(&mut self, mapping: mmu::KernelMapping)->Result<mmu::KernelMapping> {
+        self.gpu_mapping.replace(mapping).ok_or(EINVAL)
+    }
+    pub(crate) fn write(&mut self,offset:usize,data:&[u8])->Result {
+        if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let ptr=self.cpu_ptr()?;
+        unsafe {core::ptr::copy_nonoverlapping(data.as_ptr(),ptr.add(self.offset+offset),data.len())}; Ok(())
+    }
+    /// Initialize the complete owned span through its retained CPU mapping.
+    /// The caller must own the storage or have retired all device consumers.
+    pub(crate) fn fill(&mut self, value: u8) -> Result {
+        self.fill_range(0,self.size,value)
+    }
+    /// Reset one unpublished slot without touching other queued device owners.
+    pub(crate) fn fill_range(&mut self,offset:usize,size:usize,value:u8)->Result {
+        if offset.checked_add(size).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let ptr = self.cpu_ptr()?;
+        // SAFETY: allocation validates offset + size against its backing, and
+        // cpu_ptr retains that entire coherent/WC mapping for this access.
+        // Resolve the mapping once, as the G16 buffer clear path does, instead
+        // of repeating cached_vmap and memcpy for every 256-byte chunk.
+        unsafe { core::ptr::write_bytes(ptr.add(self.offset+offset), value, size) };
+        Ok(())
+    }
+    pub(crate) fn u32(&mut self,offset:usize,value:u32)->Result {
+        if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?;
+        if start&3!=0 {return Err(ERANGE);}
+        let ptr=self.cpu_ptr()?;
+        // SAFETY: aligned, bounded WC GEM storage retained by this owner.
+        unsafe {ptr.add(start).cast::<u32>().write_volatile(value.to_le())};Ok(())
+    }
+    pub(crate) fn u64(&mut self,offset:usize,value:u64)->Result {self.write(offset,&value.to_le_bytes())}
+    /// Publish the unaligned performance-request word with bounded byte stores.
+    pub(crate) fn packed_request(&mut self,offset:usize,value:u32)->Result {
+        if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let ptr=self.cpu_ptr()?;
+        // SAFETY: checked span in retained CPU storage; no aligned access is required.
+        unsafe {crate::m3::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)};
+        Ok(())
+    }
+    pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result {
+        if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let ptr=self.cpu_ptr()?;
+        // SAFETY: checked WC GEM span; firmware may change the shared bytes.
+        unsafe {
+            core::arch::asm!("dmb osh",options(nostack,preserves_flags));
+            for (i,b) in data.iter_mut().enumerate() {*b=ptr.cast_const().add(self.offset+offset+i).read_volatile();}
+        } Ok(())
+    }
+    pub(crate) fn read_u32(&mut self,offset:usize)->Result<u32> {
+        if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?;
+        if start&3!=0 {let mut b=[0;4];self.read(offset,&mut b)?;return Ok(u32::from_le_bytes(b));}
+        let ptr=self.cpu_ptr()?;
+        // SAFETY: aligned, checked atomic producer/consumer word in WC RAM.
+        let value=unsafe {ptr.cast_const().add(start).cast::<u32>().read_volatile()};
+        unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))};
+        Ok(u32::from_le(value))
+    }
+    pub(crate) fn read_u64(&mut self,offset:usize)->Result<u64> {
+        if offset.checked_add(8).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);}
+        let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?;
+        // Packed firmware records also contain unaligned u64 fields. Retain
+        // their byte-wise access rather than issuing an unaligned word load.
+        if start&7!=0 {let mut b=[0;8];self.read(offset,&mut b)?;return Ok(u64::from_le_bytes(b));}
+        let ptr=self.cpu_ptr()?;
+        // SAFETY: retained page-aligned coherent/WC backing, checked aligned
+        // eight-byte span. One volatile word load avoids eight separate reads
+        // and byte-wise tearing of an aligned firmware timestamp/counter.
+        let value=unsafe {ptr.cast_const().add(start).cast::<u64>().read_volatile()};
+        unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))};
+        Ok(u64::from_le(value))
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs
new file mode 100644
index 0000000000000..a17bc0ee399c3
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! T8122 firmware MMIO descriptors. Mapping ownership stays with the M3 runtime.
+
+use super::m3_init_layout::{Error, Region};
+
+pub(crate) const SLOTS: usize = 53;
+pub(crate) const RECORD_BYTES: usize = 40;
+pub(crate) const TABLE_OFFSET: usize = 0x640;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct Area {
+    pub(crate) physical: u64,
+    pub(crate) bytes: usize,
+    relative_register: u64,
+    writable: bool,
+    split: bool,
+}
+
+impl Area {
+    pub(crate) fn writable(self) -> bool {
+        self.writable
+    }
+
+    pub(crate) fn page_offset(self) -> usize {
+        (self.physical & 0x3fff) as usize
+    }
+
+    pub(crate) fn mapped_bytes(self) -> usize {
+        (self.page_offset() + self.bytes + 0x3fff) & !0x3fff
+    }
+
+    /// At most two physical banks; the second bank occupies the next virtual pages.
+    pub(crate) fn banks(self) -> [(u64, usize, usize); 2] {
+        if self.split {
+            [
+                (self.physical, self.bytes / 2, 0),
+                (self.physical + 0x200_0000, self.bytes / 2, self.bytes / 2),
+            ]
+        } else {
+            [(self.physical & !0x3fff, self.mapped_bytes(), 0), (0, 0, 0)]
+        }
+    }
+}
+
+/// Register resources obtained from the admitted T8122 hardware configuration.
+/// Their physical addresses are supplied by the owner, not duplicated in a wire table.
+#[derive(Clone, Copy)]
+pub(crate) struct RegisterBases {
+    pub(crate) gpu: u64,
+    pub(crate) interrupt: u64,
+    pub(crate) memory: u64,
+    pub(crate) telemetry_write: u64,
+    pub(crate) telemetry_read: u64,
+    pub(crate) neural: u64,
+    pub(crate) metrology: u64,
+}
+
+/// Select the wire slot and map its logical hardware block independently of its VA.
+pub(crate) fn area(slot: usize, bases: RegisterBases) -> Result<Option<Area>, Error> {
+    let (base, offset, bytes, tagged, writable, split) = match slot {
+        0 => (bases.interrupt, 0, 0x4000, false, true, false),
+        3 => (bases.memory, 0, 0x30000, false, true, true),
+        9 => (bases.telemetry_write, 0, 0x1000, false, true, false),
+        10 => (bases.telemetry_read, 0, 0x2000, false, false, false),
+        12 => (bases.neural, 0, 0x4000, false, true, false),
+        14 => (bases.metrology, 0, 0x8000, false, false, false),
+        17 => (bases.gpu, 0, 0x20000, false, true, false),
+        26 => (bases.gpu, 0xd04000, 0x8000, true, true, false),
+        27 => (bases.gpu, 0xd0d000, 0x1000, true, true, false),
+        28 => (bases.gpu, 0xd50000, 0x8000, true, true, false),
+        29 => (bases.gpu, 0xd10000, 0x4000, true, true, false),
+        31 => (bases.gpu, 0xd40000, 0x4000, true, true, false),
+        32 => (bases.gpu, 0xd60000, 0x8000, true, true, false),
+        35 => (bases.gpu, 0xe00000, 0x4000, true, true, false),
+        39 => (bases.gpu, 0xe08000, 0x8000, false, true, false),
+        40 => (bases.gpu, 0xe1c000, 0x4000, true, true, false),
+        0..SLOTS => return Ok(None),
+        _ => return Err(Error::Bounds),
+    };
+    let physical = base.checked_add(offset).ok_or(Error::Address)?;
+    if physical == 0 || physical.checked_add(bytes as u64).ok_or(Error::Address)? > 1 << 42 {
+        return Err(Error::Address);
+    }
+    if split
+        && (physical & 0x3fff != 0
+            || physical
+                .checked_add(0x200_0000)
+                .and_then(|v| v.checked_add((bytes / 2) as u64))
+                .filter(|v| *v <= 1 << 42)
+                .is_none())
+    {
+        return Err(Error::Address);
+    }
+    Ok(Some(Area {
+        physical,
+        bytes,
+        relative_register: if tagged { offset } else { 0 },
+        writable,
+        split,
+    }))
+}
+
+fn empty_mode(slot: usize) -> u64 {
+    if matches!(slot, 2 | 5..=8 | 30 | 33 | 34 | 37 | 38 | 41..=43 | 46..=49 | 52) {
+        2
+    } else {
+        0
+    }
+}
+
+/// Encode a complete record after checking the owned mapping; errors preserve all output.
+pub(crate) fn encode(
+    slot: usize,
+    bases: RegisterBases,
+    mapping: Option<Region>,
+    output: &mut [u8],
+) -> Result<(), Error> {
+    let descriptor = area(slot, bases)?;
+    let (physical, va, bytes, requested, register, mode) = match (descriptor, mapping) {
+        (None, None) => (0, 0, 0, 0, 0, empty_mode(slot)),
+        (Some(item), Some(mapping)) => {
+            let page = mapping.at(0, item.mapped_bytes())?;
+            if page & 0x3fff != 0 {
+                return Err(Error::Address);
+            }
+            let va = page
+                .checked_add(item.page_offset() as u64)
+                .ok_or(Error::Address)?;
+            (
+                item.physical,
+                va,
+                item.bytes as u32,
+                if item.split {
+                    (item.bytes / 2) as u32
+                } else {
+                    item.bytes as u32
+                },
+                item.relative_register,
+                if item.writable { 2 } else { 0 },
+            )
+        }
+        _ => return Err(Error::Address),
+    };
+    let bytes_out = output.get_mut(..RECORD_BYTES).ok_or(Error::Size)?;
+    bytes_out.fill(0);
+    bytes_out[0..8].copy_from_slice(&physical.to_le_bytes());
+    bytes_out[8..16].copy_from_slice(&va.to_le_bytes());
+    bytes_out[16..20].copy_from_slice(&bytes.to_le_bytes());
+    bytes_out[20..24].copy_from_slice(&requested.to_le_bytes());
+    bytes_out[24..32].copy_from_slice(&register.to_le_bytes());
+    bytes_out[32..40].copy_from_slice(&mode.to_le_bytes());
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Resource owners exercise subpage mappings and split memory-controller banks.
+    const BASES: RegisterBases = RegisterBases {
+        gpu: 0x2_9000_0000,
+        interrupt: 0x2_d101_4000,
+        memory: 0x2_201c_4000,
+        telemetry_write: 0x2_d03d_0000,
+        telemetry_read: 0x2_d03c_0000,
+        neural: 0x3_1145_c000,
+        metrology: 0x2_d028_0000,
+    };
+
+    #[test]
+    fn j613_slot28_bank_and_wire_tag_agree() {
+        let bank = area(28, BASES).unwrap().unwrap();
+        assert_eq!(bank.physical, BASES.gpu + 0xd50000);
+        assert_eq!(bank.bytes, 0x8000);
+        let mapped = Region::new(0xffff_fc23_01c0_0000, 0x8000).unwrap();
+        let mut bytes = [0; RECORD_BYTES];
+        encode(28, BASES, Some(mapped), &mut bytes).unwrap();
+        assert_eq!(&bytes[0..8], &(BASES.gpu + 0xd50000).to_le_bytes());
+        assert_eq!(&bytes[24..32], &0xd50000u64.to_le_bytes());
+        assert_ne!(bank.physical, BASES.gpu + 0xd58000);
+    }
+
+    #[test]
+    fn descriptor_inventory_ends_before_the_retention_pointer() {
+        assert_eq!(TABLE_OFFSET + SLOTS * RECORD_BYTES, 0xe88);
+        assert_eq!(
+            (0..SLOTS)
+                .filter(|slot| area(*slot, BASES).unwrap().is_some())
+                .count(),
+            16
+        );
+        assert!(area(SLOTS, BASES).is_err());
+    }
+
+    #[test]
+    fn discontiguous_memory_controller_banks_have_distinct_physical_owners() {
+        let item = area(3, BASES).unwrap().unwrap();
+        assert_eq!(
+            item.banks(),
+            [
+                (0x2_201c_4000, 0x18000, 0),
+                (0x2_221c_4000, 0x18000, 0x18000)
+            ]
+        );
+        let mapping = Region::new(0xffff_fc23_0000_0000, 0x30000).unwrap();
+        let mut bytes = [0xa5; RECORD_BYTES];
+        encode(3, BASES, Some(mapping), &mut bytes).unwrap();
+        assert_eq!(
+            u32::from_le_bytes(bytes[16..20].try_into().unwrap()),
+            0x30000
+        );
+        assert_eq!(
+            u32::from_le_bytes(bytes[20..24].try_into().unwrap()),
+            0x18000
+        );
+    }
+
+    #[test]
+    fn subpage_mapping_keeps_the_register_offset() {
+        let item = area(27, BASES).unwrap().unwrap();
+        assert_eq!(item.page_offset(), 0x1000);
+        assert_eq!(item.banks()[0], (0x2_90d0_c000, 0x4000, 0));
+        let mut output = [0xa5; RECORD_BYTES + 8];
+        encode(
+            27,
+            BASES,
+            Some(Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap()),
+            &mut output,
+        )
+        .unwrap();
+        assert_eq!(
+            u64::from_le_bytes(output[8..16].try_into().unwrap()),
+            0xffff_fc23_0000_1000
+        );
+        assert_eq!(&output[RECORD_BYTES..], &[0xa5; 8]);
+    }
+
+    #[test]
+    fn empty_slots_keep_modes_without_fabricating_mappings() {
+        for slot in 0..SLOTS {
+            if area(slot, BASES).unwrap().is_some() {
+                continue;
+            }
+            let mut output = [0xa5; RECORD_BYTES];
+            encode(slot, BASES, None, &mut output).unwrap();
+            assert_eq!(&output[..32], &[0; 32]);
+            assert_eq!(
+                u64::from_le_bytes(output[32..40].try_into().unwrap()),
+                empty_mode(slot)
+            );
+        }
+    }
+
+    #[test]
+    fn invalid_mapping_or_output_never_partially_changes_a_descriptor() {
+        let good = Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap();
+        for (slot, mapping) in [
+            (SLOTS, None),
+            (0, None),
+            (1, Some(good)),
+            (0, Some(Region::new(0xffff_fc23_0000_0001, 0x4000).unwrap())),
+            (0, Some(Region::new(0xffff_fc23_0000_0000, 0x3fff).unwrap())),
+        ] {
+            let mut output = [0xa5; RECORD_BYTES];
+            assert!(encode(slot, BASES, mapping, &mut output).is_err());
+            assert_eq!(output, [0xa5; RECORD_BYTES]);
+        }
+        for length in 0..RECORD_BYTES {
+            let mut output = [0xa5; RECORD_BYTES];
+            assert_eq!(
+                encode(0, BASES, Some(good), &mut output[..length]),
+                Err(Error::Size)
+            );
+            assert_eq!(output, [0xa5; RECORD_BYTES]);
+        }
+    }
+    #[test]
+    fn split_banks_require_page_alignment_and_both_fit_the_bus() {
+        for memory in [BASES.memory + 4, (1 << 42) - 0x100_0000] {
+            let mut bases = BASES;
+            bases.memory = memory;
+            let mut output = [0xa5; RECORD_BYTES];
+            assert!(encode(
+                3,
+                bases,
+                Some(Region::new(0xffff_fc21_0000_0000, 0x30000).unwrap()),
+                &mut output
+            )
+            .is_err());
+            assert_eq!(output, [0xa5; RECORD_BYTES]);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs
index 684505615adf3..97c1367b29455 100644
--- a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs
@@ -61,3 +61,77 @@ impl Manager {
         Ok(())
     }
 }
+
+impl Manager {
+    pub(crate) const EXTENDED_SIZE: usize = 0xc0;
+
+    /// J613's extended manager has both firmware and GPU page-list aliases.
+    /// The block ring is read through its canonical firmware mapping.
+    pub(crate) fn encode_extended(&self, pages_fw: FirmwareVa, blocks_fw: FirmwareVa,
+        hardware_id: u32, output: &mut [u8]) -> Result<(), Error> {
+        let capacity = self.block_count.checked_add(1).and_then(u32::checked_next_power_of_two)
+            .ok_or(Error::Size)?;
+        if hardware_id >= 32 || capacity as usize > self.blocks.size() / 8 {
+            return Err(Error::Size);
+        }
+        let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        let word = |bytes: &mut [u8], offset: usize, value: u32| {
+            bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
+        };
+        let address = |bytes: &mut [u8], offset: usize, value: u64| {
+            bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
+        };
+        word(bytes, 8, hardware_id);
+        word(bytes, 12, hardware_id);
+        address(bytes, 0x1c, pages_fw.get());
+        address(bytes, 0x24, self.pages.base());
+        word(bytes, 0x2c, self.pages.size() as u32);
+        word(bytes, 0x30, self.page_count);
+        word(bytes, 0x34, capacity);
+        word(bytes, 0x38, self.block_count);
+        address(bytes, 0x40, blocks_fw.get());
+        address(bytes, 0x48, self.control.get());
+        word(bytes, 0x50, self.page_count - 1);
+        word(bytes, 0x54, BLOCK_SIZE as u32);
+        address(bytes, 0x60, self.counter.get());
+        word(bytes, 0x78, self.page_count);
+        word(bytes, 0x7c, self.page_count);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    fn fixture() -> Manager {
+        Manager::new(GpuRegion::new(0x5000_0000, 0x4000).unwrap(),
+            GpuRegion::new(0x6000_0000, 0x4000).unwrap(),
+            FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(),
+            FirmwareVa::new(0xffff_fc24_0020_0000).unwrap(), 32).unwrap()
+    }
+    #[test] fn native_manager_keeps_distinct_aliases_and_hardware_identity() {
+        let mut bytes = [0xa5; Manager::EXTENDED_SIZE + 8];
+        let pages = FirmwareVa::new(0xffff_fc24_0030_0000).unwrap();
+        let blocks = FirmwareVa::new(0xffff_fc24_0040_0000).unwrap();
+        fixture().encode_extended(pages, blocks, 7, &mut bytes).unwrap();
+        let word = |at| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap());
+        let pointer = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap());
+        assert_eq!(word(8), 7); assert_eq!(word(12), 7);
+        assert_eq!(pointer(0x1c), pages.get());
+        assert_eq!(pointer(0x24), 0x5000_0000);
+        assert_eq!(pointer(0x40), blocks.get());
+        assert_eq!(word(0x34), 64); assert_eq!(word(0x38), 32);
+        assert_eq!(word(0x50), 127);
+        assert_eq!(pointer(0x80), 0);
+        assert_eq!(&bytes[Manager::EXTENDED_SIZE..], &[0xa5; 8]);
+    }
+    #[test] fn native_manager_rejects_short_storage_and_invalid_identity_before_mutation() {
+        let fw = FirmwareVa::new(0xffff_fc24_0010_0000).unwrap();
+        let mut bytes = [0xa5; Manager::EXTENDED_SIZE];
+        assert!(fixture().encode_extended(fw, fw, 32, &mut bytes).is_err());
+        assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]);
+        assert!(fixture().encode_extended(fw, fw, 0, &mut bytes[..MANAGER_SIZE]).is_err());
+        assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs
new file mode 100644
index 0000000000000..85033c88e4192
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_params.rs
@@ -0,0 +1,681 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Module parameters of the T6030 (M3) backends.
+//!
+//! The `module!` parameter support only parses plain integers. The parameters here are declared
+//! with their own `kernel_param` entries (the same layout the `module!` macro emits), so they can
+//! take symbolic values on the kernel command line, e.g. `asahi.m3_backend=manager`, and hex or
+//! decimal masks.
+
+use core::ffi::{
+    c_char,
+    c_int,
+};
+use core::sync::atomic::{
+    AtomicU64,
+    Ordering, //
+};
+
+use kernel::{
+    device::Core,
+    platform,
+    prelude::*, //
+};
+
+/// Parse a decimal or `0x`-prefixed hexadecimal `u64`.
+fn parse_u64(text: &str) -> Option<u64> {
+    let text = text.trim();
+    match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) {
+        Some(hex) => u64::from_str_radix(hex, 16).ok(),
+        None => text.parse::<u64>().ok(),
+    }
+}
+
+/// Store the parsed value of a parameter string into the `AtomicU64` behind `kp`.
+///
+/// # Safety
+///
+/// `val` must be NULL or a NUL-terminated string, and `kp` must be a `kernel_param` declared by
+/// [`m3_param!`], whose `arg` points at a static `AtomicU64`.
+unsafe fn set_param(
+    val: *const c_char,
+    kp: *const kernel::bindings::kernel_param,
+    parse: fn(&str) -> Option<u64>,
+) -> c_int {
+    if val.is_null() {
+        return EINVAL.to_errno();
+    }
+    // SAFETY: `val` is a NUL-terminated string per the function contract.
+    let Ok(text) = unsafe { core::ffi::CStr::from_ptr(val) }.to_str() else {
+        return EINVAL.to_errno();
+    };
+    let Some(value) = parse(text) else {
+        return EINVAL.to_errno();
+    };
+    // SAFETY: `arg` points at the static `AtomicU64` this entry was declared with.
+    let slot = unsafe { &*(*kp).__bindgen_anon_1.arg.cast::<AtomicU64>() };
+    slot.store(value, Ordering::Relaxed);
+    0
+}
+
+/// Declare a module parameter `asahi.<name>` stored in the static `AtomicU64` `$storage` and
+/// parsed by `$parse: fn(&str) -> Option<u64>`. Not visible in sysfs.
+macro_rules! m3_param {
+    ($name:literal, $storage:ident, $parse:path) => {
+        m3_param!($name, $storage, $parse, 0, None);
+    };
+    ($name:literal, $storage:ident, $parse:path, $perm:expr, $get:expr) => {
+        const _: () = {
+            unsafe extern "C" fn set(
+                val: *const c_char,
+                kp: *const kernel::bindings::kernel_param,
+            ) -> c_int {
+                // SAFETY: Called by the parameter parser with a value string (or NULL) and the
+                // `kernel_param` entry declared below.
+                unsafe { set_param(val, kp, $parse) }
+            }
+
+            static OPS: kernel::bindings::kernel_param_ops = kernel::bindings::kernel_param_ops {
+                flags: 0,
+                set: Some(set),
+                get: $get,
+                free: None,
+            };
+
+            #[link_section = "__param"]
+            #[used(compiler)]
+            static PARAM: kernel::module_param::KernelParam =
+                kernel::module_param::KernelParam::new(kernel::bindings::kernel_param {
+                    name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) {
+                        kernel::c_str!($name)
+                    } else {
+                        kernel::c_str!(concat!("asahi.", $name))
+                    }),
+                    // `__this_module` is constructed by the kernel at load time and is not
+                    // freed until the module is unloaded. Take its address without creating a
+                    // reference to the mutable static.
+                    #[cfg(MODULE)]
+                    mod_: (&raw const kernel::bindings::__this_module).cast_mut(),
+                    #[cfg(not(MODULE))]
+                    mod_: core::ptr::null_mut(),
+                    ops: core::ptr::from_ref(&OPS),
+                    perm: $perm,
+                    level: -1,
+                    flags: 0,
+                    __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 {
+                        arg: core::ptr::from_ref(&$storage).cast_mut().cast(),
+                    },
+                });
+        };
+    };
+}
+
+/// Read an atomic parameter without borrowing storage that a sysfs write may
+/// change concurrently. The module-parameter core supplies a PAGE_SIZE buffer.
+///
+/// # Safety
+///
+/// `buffer` must be the module-parameter core's writable output buffer and
+/// `kp` must be an entry declared by `m3_param!` with AtomicU64 storage.
+unsafe extern "C" fn get_atomic_param(
+    buffer: *mut c_char,
+    kp: *const kernel::bindings::kernel_param,
+) -> c_int {
+    use core::fmt::Write;
+
+    // SAFETY: The entry points to static AtomicU64 storage by contract. Loads
+    // and stores are atomic; module-parameter serialization alone would not
+    // protect the scheduler's concurrent reads.
+    let value = unsafe { &*((*kp).__bindgen_anon_1.arg.cast::<AtomicU64>()) }
+        .load(Ordering::Relaxed);
+    // SAFETY: 32 bytes fit in the core's PAGE_SIZE output buffer and hold any
+    // decimal u64 plus its NUL. The core appends the sysfs newline itself.
+    let bytes = unsafe { core::slice::from_raw_parts_mut(buffer.cast::<u8>(), 32) };
+    let mut output = kernel::str::Formatter::new(bytes);
+    if write!(output, "{}\0", value).is_err() {
+        return EINVAL.to_errno();
+    }
+    (output.bytes_written() - 1) as c_int
+}
+
+fn parse_render_batch_override(text: &str) -> Option<u64> {
+    parse_u64(text).filter(|value| *value <= crate::m3::m3_pass_layout::SLOTS as u64)
+}
+
+/// Runtime tuning is opt-in: zero keeps the existing boot parameter and its
+/// clamping semantics. Only root may write a bounded 1..=SLOTS override; an
+/// invalid write leaves the previous value intact. Compute batching is separate.
+static M3_RENDER_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_render_batch_override", M3_RENDER_BATCH_OVERRIDE,
+    parse_render_batch_override, 0o644, Some(get_atomic_param));
+
+fn parse_render_stamp_coalesce(text:&str)->Option<u64> {
+    parse_u64(text).filter(|value| *value<=1)
+}
+
+/// Root-only T6030 render stamp-flush coalescing; zero retains per-pass flushes.
+/// Each publication snapshots the setting before encoding any pass.
+static M3_RENDER_STAMP_COALESCE: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_render_stamp_coalesce", M3_RENDER_STAMP_COALESCE,
+    parse_render_stamp_coalesce, 0o644, Some(get_atomic_param));
+pub(crate) fn render_stamp_coalesce()->bool {
+    M3_RENDER_STAMP_COALESCE.load(Ordering::Relaxed)!=0
+}
+
+/// T6030 cross-engine publications retain GPU ordering with firmware barriers.
+/// Zero preserves the existing host retirement between engine groups.
+static M3_MIXED_BATCH: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_mixed_batch", M3_MIXED_BATCH,
+    parse_render_stamp_coalesce, 0o644, Some(get_atomic_param));
+pub(crate) fn mixed_batch()->bool {
+    M3_MIXED_BATCH.load(Ordering::Relaxed)!=0
+}
+
+/// Boot-only T6030 power-state sampling while GPU stamps are outstanding.
+/// Zero preserves the completion path without additional register reads.
+static M3_ACTIVE_POWER: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_active_power", M3_ACTIVE_POWER, parse_render_stamp_coalesce);
+pub(crate) fn active_power()->bool {
+    M3_ACTIVE_POWER.load(Ordering::Relaxed)==1
+}
+
+/// Boot-only T6030 Globals debug-word experiment. Zero preserves the runtime word.
+static M3_GLOBALS_DEBUG_OFF: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_globals_debug_off",M3_GLOBALS_DEBUG_OFF,parse_render_stamp_coalesce);
+pub(crate) fn globals_debug_off()->bool {M3_GLOBALS_DEBUG_OFF.load(Ordering::Relaxed)==1}
+
+/// T6030-only request while work is pending or active. Zero leaves firmware
+/// performance selection unchanged; three requests the top published state.
+static M3_CONSISTENT_PERF: AtomicU64 = AtomicU64::new(0);
+fn parse_consistent_perf(text: &str)->Option<u64> {
+    parse_u64(text).filter(|v| matches!(*v,0|3))
+}
+m3_param!("m3_consistent_perf",M3_CONSISTENT_PERF,parse_consistent_perf,0o644,Some(get_atomic_param));
+pub(crate) fn consistent_perf()->u64 {M3_CONSISTENT_PERF.load(Ordering::Relaxed)}
+
+/// One preserves internal render measurements. Application timestamp requests
+/// remain recorded when zero omits measurements on T6030.
+static M3_RENDER_INTERNAL_TIMESTAMPS: AtomicU64 = AtomicU64::new(1);
+m3_param!("m3_render_internal_timestamps", M3_RENDER_INTERNAL_TIMESTAMPS,
+    parse_render_stamp_coalesce, 0o644, Some(get_atomic_param));
+pub(crate) fn render_internal_timestamps()->bool {
+    M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0
+}
+
+// The runtime's boot-time module parameters. On 7.1.12 they were `module!` parameters; they
+// keep their names, and exist only with the T8122 runtime built in.
+
+/// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16).
+static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1);
+m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64);
+/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16).
+static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1);
+m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64);
+/// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies.
+static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_early_tiling", M3_EARLY_TILING, parse_u64);
+
+/// `asahi.m3_early_tiling` is set.
+pub(crate) fn early_tiling() -> bool {
+    M3_EARLY_TILING.load(Ordering::Relaxed) != 0
+}
+
+fn parse_expose(text: &str) -> Option<u64> {
+    match text.trim() {
+        "-1" => Some(EXPOSE_AUTO),
+        other => parse_u64(other).filter(|v| *v <= 1),
+    }
+}
+const EXPOSE_AUTO: u64 = u64::MAX;
+/// `asahi.m3_expose`: M3 render node registration: -1 = auto (on the boards where the runtime
+/// is validated), 0 = never, 1 = always.
+static M3_EXPOSE: AtomicU64 = AtomicU64::new(EXPOSE_AUTO);
+m3_param!("m3_expose", M3_EXPOSE, parse_expose);
+
+/// `asahi.m3_expose`: `None` for auto, else whether to register the render node.
+pub(crate) fn expose() -> Option<bool> {
+    match M3_EXPOSE.load(Ordering::Relaxed) {
+        EXPOSE_AUTO => None,
+        v => Some(v != 0),
+    }
+}
+
+/// Snapshot once per batch. A live change affects later publications, never the
+/// storage, ordering or retirement checks of work already on the firmware queues.
+pub(crate) fn render_batch_size() -> usize {
+    let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed);
+    let size = if override_size == 0 {
+        M3_RENDER_BATCH_SIZE.load(Ordering::Relaxed) as usize
+    } else {
+        override_size as usize
+    };
+    size.clamp(1, crate::m3::m3_pass_layout::SLOTS)
+}
+
+fn parse_compute_batch_override(text: &str) -> Option<u64> {
+    parse_u64(text).filter(|value| *value <= crate::m3::m3_compute_storage::SLOTS as u64)
+}
+
+/// Independently tune the existing adjacent-compute path. Zero preserves the
+/// boot parameter; invalid writes do not replace the current value. Storage and
+/// engine/VM retirement boundaries are unchanged.
+static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE,
+    parse_compute_batch_override, 0o644, Some(get_atomic_param));
+
+/// Snapshot once per packet, so a live write never changes an in-flight batch.
+pub(crate) fn compute_batch_size() -> usize {
+    let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed);
+    let size = if override_size == 0 {
+        M3_COMPUTE_BATCH_SIZE.load(Ordering::Relaxed) as usize
+    } else {
+        override_size as usize
+    };
+    size.clamp(1, crate::m3::m3_compute_storage::SLOTS)
+}
+
+/// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`].
+static G15_DEBUG: AtomicU64 = AtomicU64::new(0);
+m3_param!("g15_debug", G15_DEBUG, parse_u64);
+
+/// Bits of `asahi.g15_debug`, the bring-up switches of the G15 manager backend.
+///
+/// The bit numbers are unchanged from their earlier `asahi.debug_flags` positions. Bits 49
+/// and 50 are no longer switches: the HwDataB power-management flags they enabled are always
+/// set on G15.
+#[derive(Copy, Clone)]
+#[repr(u32)]
+pub(crate) enum G15Debug {
+    /// Debug switch: issue the Fender kick (0x11, then 0x10) before starting the ASC. The
+    /// G15 firmware boots and runs jobs without it.
+    FenderKick = 41,
+    /// Register the DRM device (card/render nodes) for the manager backend, like
+    /// `asahi.m3_expose=1`. Off by default so compositors and system Mesa never see a GPU that
+    /// cannot run userspace jobs yet; the firmware bring-up runs either way.
+    ExposeDrm = 42,
+    /// Turn the firmware log on (RuntimePointers+0x230 = 1) before MSG_INIT. The firmware
+    /// writes no log records while it is 0.
+    FwLog = 43,
+    /// A/B: ring a DEVCTRL (0x83|0x11) and a KICKFW (0x83|0x10) doorbell immediately after
+    /// MSG_INIT, before the firmware has registered its EP 0x21 handler.
+    LegacyDoorbells = 44,
+    /// Skip the post-init DeviceControl liveness probe (no doorbell at all after MSG_INIT).
+    NoDevctlProbe = 45,
+    /// Map the gfx-data carveout (firmware data) read-only and log the init progress words,
+    /// thread handles and crashlog fill at every health sample.
+    GfxDataPeek = 46,
+    /// After firmware init, submit one empty compute job and poll its stamp for 2 seconds.
+    /// Never fails probe and never blocks boot past that poll.
+    SelfTest = 47,
+    /// With [`G15Debug::SelfTest`]: first run a firmware-only copy of the job (cmd+0x878 = 1,
+    /// StartCL skip path, no hardware kick). The real job runs only if that one completes.
+    SelfTestSkipFirst = 51,
+    /// Runtime backend: stop the probe right before the GPU coprocessor would be started. The
+    /// firmware never runs; everything before that point (admission, firmware identity,
+    /// InitData generation, register identity checks) still does.
+    StopBeforeAsc = 53,
+    /// A/B for the manager backend: publish the performance states as the two voltage-sorted
+    /// tables (with their device-tree index maps) instead of one table in device-tree order.
+    ManagerSplitPstates = 54,
+    /// A/B for the manager backend: write the power-controller block's PPM words at +0xc8..+0xd8
+    /// in the order the runtime backend's InitData uses (0, enable, target power, kp, ki*dt),
+    /// and the Globals power-interface targets and performance-state cap, instead of the
+    /// manager's validated values.
+    ManagerReferencePpm = 55,
+    /// Debug switch for the runtime backend: when a job fails and the firmware reports that it has
+    /// halted, clear the halted flag and set resume, then log for about half a second what the
+    /// firmware, the engines and the failed job do. The GPU stays marked failed either way.
+    M3ResumeAfterFault = 56,
+}
+
+/// Returns whether a `asahi.g15_debug` bit is set.
+pub(crate) fn g15_debug(bit: G15Debug) -> bool {
+    G15_DEBUG.load(Ordering::Relaxed) & (1u64 << (bit as u32)) != 0
+}
+
+/// The raw `asahi.g15_debug` value, for logging.
+pub(crate) fn g15_debug_mask() -> u64 {
+    G15_DEBUG.load(Ordering::Relaxed)
+}
+
+/// `asahi.m3_backend` values.
+const BACKEND_AUTO: u64 = 0;
+const BACKEND_RUNTIME: u64 = 1;
+const BACKEND_MANAGER: u64 = 2;
+const BACKEND_OFF: u64 = 3;
+
+fn parse_backend(text: &str) -> Option<u64> {
+    match text.trim() {
+        "auto" | "0" => Some(BACKEND_AUTO),
+        "runtime" | "1" => Some(BACKEND_RUNTIME),
+        "manager" | "2" => Some(BACKEND_MANAGER),
+        "off" | "3" => Some(BACKEND_OFF),
+        _ => None,
+    }
+}
+
+/// `asahi.m3_backend=auto|runtime|manager|off`: which T6030 GPU backend probe starts.
+static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO);
+m3_param!("m3_backend", M3_BACKEND, parse_backend);
+
+
+/// `asahi.m3_initdata` values.
+const INITDATA_AUTO: u64 = 0;
+const INITDATA_ADT: u64 = 2;
+
+fn parse_initdata(text: &str) -> Option<u64> {
+    match text.trim() {
+        "auto" | "0" => Some(INITDATA_AUTO),
+        "adt" | "2" => Some(INITDATA_ADT),
+        _ => None,
+    }
+}
+
+/// `asahi.m3_initdata=auto|adt`: the runtime backend's InitData contents are generated from the
+/// device tree either way; the value is logged.
+static M3_INITDATA: AtomicU64 = AtomicU64::new(INITDATA_AUTO);
+m3_param!("m3_initdata", M3_INITDATA, parse_initdata);
+
+/// Requested source of the runtime backend's InitData contents.
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub(crate) enum InitDataParam {
+    /// Not given: contents generated from the device tree.
+    Auto,
+    /// Contents generated from the device tree.
+    Adt,
+}
+
+/// The `asahi.m3_initdata` setting.
+pub(crate) fn initdata_param() -> InitDataParam {
+    match M3_INITDATA.load(Ordering::Relaxed) {
+        INITDATA_ADT => InitDataParam::Adt,
+        _ => InitDataParam::Auto,
+    }
+}
+
+/// Default of `asahi.m3_max_pstate` for device-tree InitData and the G15 manager backend: the
+/// fifth performance state (1056 MHz on J516S).
+pub(crate) const MAX_PSTATE_DEFAULT: u64 = 5;
+/// A performance-state parameter that is not given.
+const PSTATE_UNSET: u64 = u64::MAX;
+
+/// `asahi.m3_max_pstate`: the highest GPU performance state the driver lets the firmware use,
+/// as an index into the voltage-sorted performance-state table (1 = lowest, 8 = highest on
+/// T6030; the G15 manager backend translates it to its device-tree ordered table). It is
+/// clamped to the device tree's highest performance state. Unset means
+/// [`MAX_PSTATE_DEFAULT`], except with the runtime backend while its thermal limit is on or
+/// holding, which keeps the whole table.
+static M3_MAX_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET);
+m3_param!("m3_max_pstate", M3_MAX_PSTATE, parse_u64);
+
+/// `asahi.m3_boot_pstate`: the performance state the runtime backend requests at boot, clamped
+/// to the cap. Unset means the cap.
+static M3_BOOT_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET);
+m3_param!("m3_boot_pstate", M3_BOOT_PSTATE, parse_u64);
+
+/// The `asahi.m3_max_pstate` setting, if given, before clamping to the performance-state table.
+pub(crate) fn max_pstate_param() -> Option<u64> {
+    Some(M3_MAX_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET)
+}
+
+/// The `asahi.m3_boot_pstate` setting, if given, before clamping.
+pub(crate) fn boot_pstate_param() -> Option<u64> {
+    Some(M3_BOOT_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET)
+}
+
+/// `asahi.m3_timeout_nohang`: 1 (default) makes a scheduler timeout of a runtime-backend job
+/// report "no hang", because the runtime completes every packet itself and enforces its own
+/// per-batch completion limit. 0 keeps the previous handling: mark the GPU failed, fail the
+/// packet again and report the device as gone.
+static M3_TIMEOUT_NOHANG: AtomicU64 = AtomicU64::new(1);
+m3_param!("m3_timeout_nohang", M3_TIMEOUT_NOHANG, parse_u64);
+
+/// Whether `asahi.m3_timeout_nohang` is set (the default).
+pub(crate) fn timeout_nohang() -> bool {
+    M3_TIMEOUT_NOHANG.load(Ordering::Relaxed) != 0
+}
+
+/// `asahi.m3_unlocked_wait`: 1 (default) releases the runtime lock while the runtime backend
+/// waits for a batch to complete, so VM creation, timestamp mapping and the event worker are
+/// not held up by a running job. 0 holds the lock from preparation to retirement, as before.
+static M3_UNLOCKED_WAIT: AtomicU64 = AtomicU64::new(1);
+m3_param!("m3_unlocked_wait", M3_UNLOCKED_WAIT, parse_u64);
+
+/// Whether `asahi.m3_unlocked_wait` is set (the default).
+pub(crate) fn unlocked_wait() -> bool {
+    M3_UNLOCKED_WAIT.load(Ordering::Relaxed) != 0
+}
+
+/// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback
+/// cap. Unset: the SoC's default ([`crate::m3::m3_soc::Soc::ut_engagement`]).
+static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET);
+const UT_ENGAGEMENT_UNSET: u64 = u64::MAX;
+fn parse_ut_engagement(text: &str) -> Option<u64> {
+    parse_u64(text).filter(|v| *v <= 1)
+}
+m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement);
+
+/// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it.
+pub(crate) fn ut_engagement(soc: &crate::m3::m3_soc::Soc) -> (u32, bool) {
+    match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) {
+        UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false),
+        v => (v as u32, true),
+    }
+}
+
+/// `asahi.m3_pipeline_depth`: writable at runtime. How many render batches may be in flight on
+/// the firmware's render queues. Zero selects the SoC default: 2 on the explicitly enabled
+/// experimental T8122 runtime, 1 on T6030. Depth 1 publishes only after the previous batch has
+/// fully retired. Larger values publish the next render batch of the same VM while earlier
+/// ones still run, in disjoint pass slots, ordered by the existing cross-pass barriers.
+static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0);
+m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param));
+
+fn parse_pipeline_depth(text: &str) -> Option<u64> {
+    parse_u64(text).filter(|v| *v<=crate::m3::m3_pass_layout::SLOTS as u64)
+}
+
+/// Effective depth; changing it never releases already published pass slots.
+pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize {
+    match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) {
+        0 => crate::m3::m3_runtime::policy::default_depth(crate::m3::t8122_start::is_t8122(soc)),
+        v => v.clamp(1,crate::m3::m3_pass_layout::SLOTS as u64) as usize,
+    }
+}
+
+/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets
+/// of its VM only while the VM's measured GPU time per pass, times the passes already in the
+/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro).
+/// Coalescing a client's heavy frames
+/// otherwise makes every other client's work, the compositor's included, wait for all of them.
+/// 0 disables the budget; u64::MAX restores the SoC default.
+static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX);
+m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param));
+
+/// The render batch GPU-time budget in ns, or `None` without one.
+pub(crate) fn render_batch_budget_ns(soc: &crate::m3::m3_soc::Soc) -> Option<u64> {
+    let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed);
+    let us=if value==u64::MAX {crate::m3::m3_runtime::policy::default_budget_us(crate::m3::t8122_start::is_t8122(soc))} else {value};
+    match us {
+        0 => None,
+        us => Some(us.saturating_mul(1000)),
+    }
+}
+
+/// Bits of `asahi.m3_retire_mmio`.
+pub(crate) const RETIRE_MMIO_BUSY: u64 = 1 << 0;
+pub(crate) const RETIRE_MMIO_FAULTS: u64 = 1 << 1;
+pub(crate) const RETIRE_MMIO_PSTATE: u64 = 1 << 2;
+const RETIRE_MMIO_UNSET: u64 = u64::MAX;
+
+fn parse_retire_mmio(text: &str) -> Option<u64> {
+    parse_u64(text).filter(|v| *v <= 7)
+}
+
+/// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every
+/// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1
+/// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance
+/// state (0xe01000). Unset: the SoC's default ([`crate::m3::m3_soc::Soc::retire_mmio`]).
+static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET);
+m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param));
+
+/// The `asahi.m3_retire_mmio` mask for `soc`.
+pub(crate) fn retire_mmio(soc: &crate::m3::m3_soc::Soc) -> u64 {
+    match M3_RETIRE_MMIO.load(Ordering::Relaxed) {
+        RETIRE_MMIO_UNSET => soc.retire_mmio,
+        mask => mask,
+    }
+}
+
+/// `asahi.m3_thermal` values.
+const THERMAL_OFF: u64 = 0;
+const THERMAL_HOLD: u64 = 1;
+const THERMAL_ON: u64 = 2;
+const THERMAL_UNSET: u64 = u64::MAX;
+
+fn parse_thermal(text: &str) -> Option<u64> {
+    match text.trim() {
+        "off" | "0" => Some(THERMAL_OFF),
+        "hold" => Some(THERMAL_HOLD),
+        "on" | "1" => Some(THERMAL_ON),
+        _ => None,
+    }
+}
+
+static M3_THERMAL: AtomicU64 = AtomicU64::new(THERMAL_UNSET);
+m3_param!("m3_thermal", M3_THERMAL, parse_thermal);
+
+/// How the runtime backend limits the GPU performance state by temperature.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub(crate) enum ThermalMode {
+    /// Today's fixed cap: the firmware is given the cap as its highest state, and nothing
+    /// changes it at runtime.
+    Off,
+    /// The firmware is given the whole table as its highest state, and the runtime cap words
+    /// hold the fixed cap for the whole boot. The same performance as `Off`, if the firmware
+    /// follows the runtime cap: this checks that it does.
+    Hold,
+    /// As `Hold`, and the runtime cap follows the SoC die temperature, up to the whole table
+    /// while the die is cool.
+    On,
+}
+
+/// The `asahi.m3_thermal` setting, if given.
+pub(crate) fn thermal_param() -> Option<ThermalMode> {
+    match M3_THERMAL.load(Ordering::Relaxed) {
+        THERMAL_OFF => Some(ThermalMode::Off),
+        THERMAL_HOLD => Some(ThermalMode::Hold),
+        THERMAL_ON => Some(ThermalMode::On),
+        _ => None,
+    }
+}
+
+/// Default and bounds of `asahi.m3_thermal_hot`, in degrees Celsius.
+pub(crate) const THERMAL_HOT_DEFAULT: u64 = 80;
+pub(crate) const THERMAL_HOT_MIN: u64 = 40;
+pub(crate) const THERMAL_HOT_MAX: u64 = 85;
+
+/// `asahi.m3_thermal_hot`: the SoC die temperature (degrees Celsius) at or above which the thermal
+/// limit lowers the runtime cap, clamped to [`THERMAL_HOT_MIN`]..=[`THERMAL_HOT_MAX`].
+static M3_THERMAL_HOT: AtomicU64 = AtomicU64::new(THERMAL_HOT_DEFAULT);
+m3_param!("m3_thermal_hot", M3_THERMAL_HOT, parse_u64);
+
+/// The `asahi.m3_thermal_hot` setting and whether it was clamped.
+pub(crate) fn thermal_hot_param() -> (u64, bool) {
+    let value = M3_THERMAL_HOT.load(Ordering::Relaxed);
+    let clamped = value.clamp(THERMAL_HOT_MIN, THERMAL_HOT_MAX);
+    (clamped, clamped != value)
+}
+
+/// Whether to start the runtime backend on a T8122 (`asahi.m3_backend`), and say why not.
+///
+/// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with
+/// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another
+/// board. The G15 manager backend is T6030-only.
+pub(crate) fn t8122_backend(pdev: &platform::Device<Core<'_>>) -> bool {
+    let dev = pdev.as_ref();
+    match M3_BACKEND.load(Ordering::Relaxed) {
+        BACKEND_OFF => {
+            dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n");
+            false
+        }
+        BACKEND_MANAGER => {
+            dev_info!(
+                dev,
+                "M3: asahi.m3_backend=manager: the G15 manager backend is T6030-only; no GPU backend started\n"
+            );
+            false
+        }
+        _ if crate::m3::m3_board::runtime_validated_board(&crate::m3::m3_soc::T8122) => true,
+        _ => {
+            dev_info!(
+                dev,
+                "M3: no GPU backend on this T8122 board: the runtime backend may start only on the M3 MacBook Airs (J613, J615)\n"
+            );
+            false
+        }
+    }
+}
+
+
+fn parse_t8122_number(text: &str) -> Option<u64> {
+    Some(crate::m3::t8122_knobs::parse_number(text))
+}
+
+fn parse_t8122_fender(text: &str) -> Option<u64> {
+    Some(crate::m3::t8122_knobs::parse_fender(text))
+}
+
+fn parse_t8122_clkgen(text: &str) -> Option<u64> {
+    Some(crate::m3::t8122_knobs::parse_clkgen(text))
+}
+
+fn parse_t8122_sgx_setup(text: &str) -> Option<u64> {
+    Some(crate::m3::t8122_knobs::parse_sgx_setup(text))
+}
+
+// The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a
+// value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather
+// than fall back to a default.
+use crate::m3::t8122_knobs::UNSET as T8122_UNSET;
+
+/// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values.
+/// Only with a boot loader that handed the GPU over.
+static T8122_START: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_start", T8122_START, parse_t8122_number);
+/// `asahi.t8122_initdata_version=<u64>`: the InitData version given to the T8122 firmware.
+static T8122_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_initdata_version", T8122_INITDATA_VERSION, parse_t8122_number);
+/// `asahi.t8122_fender=0x104000|0x12c000` (`rule`, `adt`): the Fender window size.
+static T8122_FENDER: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_fender", T8122_FENDER, parse_t8122_fender);
+/// `asahi.t8122_clkgen=e1c|e5c|none`: the GPU clock-generator IO mapping.
+static T8122_CLKGEN: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_clkgen", T8122_CLKGEN, parse_t8122_clkgen);
+/// `asahi.t8122_sgx_setup=none|t6030`: the SGX write made before the firmware starts.
+static T8122_SGX_SETUP: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_sgx_setup", T8122_SGX_SETUP, parse_t8122_sgx_setup);
+/// `asahi.t8122_unit_mask_a=<u64>`: HwDataB unit mask A (+0x17c0).
+static T8122_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_unit_mask_a", T8122_UNIT_MASK_A, parse_t8122_number);
+/// `asahi.t8122_unit_mask_b=<u32>`: HwDataB unit mask B (+0x17c8).
+static T8122_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number);
+/// `asahi.t8122_pstate_cap=N`: the highest performance state the T8122 firmware may use.
+static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET);
+m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number);
+
+/// The T8122 start-experiment parameters of this boot, as given.
+pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw {
+    let get = |p: &AtomicU64| p.load(Ordering::Relaxed);
+    crate::m3::t8122_knobs::Raw {
+        start: get(&T8122_START),
+        initdata_version: get(&T8122_INITDATA_VERSION),
+        fender: get(&T8122_FENDER),
+        clkgen: get(&T8122_CLKGEN),
+        sgx_setup: get(&T8122_SGX_SETUP),
+        unit_mask_a: get(&T8122_UNIT_MASK_A),
+        unit_mask_b: get(&T8122_UNIT_MASK_B),
+        pstate_cap: get(&T8122_PSTATE_CAP),
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs
new file mode 100644
index 0000000000000..9c0eb070039cd
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Private storage for one retired/unpublished G15 render pass. A Pass retains
+//! all its buffers and mappings until Render retires the shared queues.
+use kernel::prelude::*;
+use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_pass_layout as layout};
+use layout::{Field,Space};
+
+pub(crate) struct Pass {buffers:KVec<Buffer>}
+impl Pass {
+    pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,
+        slot:usize,clusters:u32,profile:super::m3_profile::Profile,aliases:&mut KVec<mmu::KernelMapping>)->Result<Self> {
+        let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?;
+        for field in layout::FIELDS {
+            let a=layout::allocation_for(profile,slot,field,clusters).map_err(|_|EINVAL)?;
+            let mut b=match a.space {
+                Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None,
+                    a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?,
+                Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),
+                    a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address),
+                    a.size,prot::Prot::from_pte(a.access.pte()),prot::PROT_GPU_SHARED_RW)?,
+            };
+            if a.gpu_alias.is_some() {aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?;}
+            buffers.push(b,GFP_KERNEL)?;
+        }
+        Ok(Self{buffers})
+    }
+    pub(crate) fn get(&self,field:Field)->&Buffer {&self.buffers[field as usize]}
+    pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]}
+    /// Caller must prove retirement, and must not clear a prepared sibling.
+    /// Dependency commands are rewritten completely by the sync encoder.
+    pub(crate) fn reset(&mut self,geometry:&crate::m3::agx_render::Geometry)->Result {
+        // Tiler arenas grow to retain the largest prior target. The firmware
+        // descriptors use the current geometry's strides and layer count, so
+        // only that prefix is addressable by this pass. Clear it on every use,
+        // including when growing again after a smaller pass; the unused tail
+        // may still contain retired data and is never assumed to be zero.
+        for field in &layout::FIELDS[..layout::RESET_COUNT] {
+            let bytes=match field {
+                Field::Tpc=>usize::try_from(geometry.tpc_bytes).map_err(|_|EOVERFLOW)?,
+                Field::Tilemap=>usize::try_from(geometry.tilemap_bytes).map_err(|_|EOVERFLOW)?,
+                _=>self.get(*field).size(),
+            };
+            self.get_mut(*field).fill_range(0,bytes,0)?;
+        }
+        Ok(())
+    }
+}
+/// Cached client views retain their resource identity across switches. A pass
+/// field cannot accidentally name a shared allocator or another pass's field.
+#[derive(Clone,Copy)]
+pub(crate) enum ViewOwner {Shared(usize),Pass{slot:usize,field:Field}}
diff --git a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs
index 5ace477a44fff..f88a2a2c9b4a5 100644
--- a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs
@@ -219,3 +219,39 @@ mod tests {
         }
     }
 }
+
+pub(crate) fn allocation_for(profile: super::m3_profile::Profile, slot:usize, field:Field, clusters:u32)
+    -> Result<Allocation,Error> {
+    let mut owner = board_allocation(slot,field,clusters)?;
+    if profile == super::m3_profile::Profile::Legacy { return Ok(owner); }
+    if owner.space == Space::Firmware {
+        owner.address = 0xffff_fc25_0000_0000 + (slot * COUNT + field as usize) as u64 * 0x10_0000;
+    }
+    match field {
+        Field::Scene => owner.size = 0x400,
+        Field::TilerCommand | Field::FragmentCommand => {
+            let fragment = field == Field::FragmentCommand;
+            owner.size = if fragment {0xca0} else {0x940};
+            owner.gpu_alias = Some(0x72_1000_0000 + slot as u64 * 0x4_0000
+                + u64::from(fragment) * 0x4000);
+        }
+        Field::TilerToFragment | Field::TilerDependency | Field::FragmentDependency => owner.size = 0x40,
+        _ => {}
+    }
+    Ok(owner)
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    #[test] fn extended_command_aliases_keep_whole_disjoint_pages() {
+        for slot in 0..SLOTS {
+            for field in [Field::TilerCommand,Field::FragmentCommand] {
+                let owner = allocation_for(super::super::m3_profile::Profile::Extended,slot,field,1).unwrap();
+                assert_eq!(owner.address & 0x3fff,0);
+                assert_eq!(owner.gpu_alias.unwrap() & 0x3fff,0);
+                assert!(owner.size <= 0x4000);
+            }
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs
new file mode 100644
index 0000000000000..4eb284e1291bd
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Construct a firmware-owned UMA allocator once from its graph's allocations.
+//! GPU mappings and counters outlive this initialization and remain owned by
+//! Render/Compute. Callers must never invoke this on a live or replayed pool.
+use kernel::prelude::*;
+use crate::{m3::m3_memory::Buffer, m3::m3_pool_layout as wire, m3::m3_queue_layout::FirmwareVa};
+pub(crate) struct Owners {
+    pub(crate) backing: core::ops::Range<usize>,
+    pub(crate) pages: usize,
+    pub(crate) table: usize,
+    pub(crate) manager: usize,
+    pub(crate) counter: usize,
+    pub(crate) table_index: u8,
+}
+impl Owners {
+    pub(crate) fn initialize(&self, objects: &mut [Buffer]) -> Result {
+        let buffers = objects.get(self.backing.clone()).ok_or(EINVAL)?;
+        let mut backings = KVec::with_capacity(buffers.len(), GFP_KERNEL)?;
+        for buffer in buffers {
+            backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?;
+        }
+        let pages = objects.get(self.pages).ok_or(EINVAL)?;
+        let pages = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?;
+        let table = objects.get(self.table).ok_or(EINVAL)?;
+        let table = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?;
+        let counter = FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?;
+        if objects.get(self.manager).ok_or(EINVAL)?.size() < wire::MANAGER_SIZE { return Err(EINVAL); }
+        let pool = wire::Pool::new(&backings, pages, table, counter, self.table_index).map_err(|_| EINVAL)?;
+        let mut manager = [0; wire::MANAGER_SIZE];
+        pool.encode(&mut manager).map_err(|_| EINVAL)?;
+        // Unpopulated entries remain explicitly zero from the fresh DMA owner.
+        for page in 0..wire::INITIAL_PAGES {
+            objects[self.pages].u64(page * 8, pool.page(page).map_err(|_| EINVAL)?)?;
+        }
+        for entry in 0..backings.len() {
+            objects[self.table].write(entry * wire::POOL_ENTRY_SIZE,
+                &pool.table_entry(entry).map_err(|_| EINVAL)?)?;
+        }
+        objects[self.manager].write(0, &manager)
+    }
+}
+
+impl Owners {
+    pub(crate) fn initialize_extended(&self, objects: &mut [Buffer], slot: u8,
+        compute_only: bool) -> Result {
+        let mut backings = KVec::new();
+        for index in self.backing.clone().take(8) {
+            let buffer = objects.get(index).ok_or(EINVAL)?;
+            backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?;
+        }
+        let backing = wire::ModernBacking(&backings);
+        let populated = backing.pages().map_err(|_| EINVAL)?;
+        let pages = objects.get(self.pages).ok_or(EINVAL)?;
+        let entries = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?;
+        let table = objects.get(self.table).ok_or(EINVAL)?;
+        let groups = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?;
+        if groups.size() < populated / 512 * 64 { return Err(EINVAL); }
+        let descriptor = wire::ModernPool { entries, groups,
+            submitted: FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?,
+            identity: FirmwareVa::new(objects.get(self.manager).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?,
+            slot, capacity: (entries.size() / 8).try_into()?, populated: populated.try_into()?, compute_only };
+        let mut bytes = [0; wire::ModernPool::SIZE];
+        descriptor.encode(&mut bytes).map_err(|_| EINVAL)?;
+        for index in 0..populated {
+            objects[self.pages].u64(index * 8, backing.page(index).map_err(|_| EINVAL)?)?;
+        }
+        for index in 0..populated / 512 {
+            objects[self.table].write(index * 64, &backing.group(index).map_err(|_| EINVAL)?)?;
+        }
+        objects[self.manager].write(0, &bytes)
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs
index a8474d262bc26..ac7e6771f7880 100644
--- a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs
@@ -69,3 +69,131 @@ impl<'a> Pool<'a> {
         Ok(())
     }
 }
+
+/// Extended firmware's private-memory pool descriptor. The owners and initial
+/// population are supplied separately from the descriptor's firmware state.
+pub(crate) struct ModernPool {
+    pub(crate) identity: FirmwareVa,
+    pub(crate) slot: u8,
+    pub(crate) entries: GpuRegion,
+    pub(crate) groups: GpuRegion,
+    pub(crate) submitted: FirmwareVa,
+    pub(crate) capacity: u32,
+    pub(crate) populated: u32,
+    pub(crate) compute_only: bool,
+}
+
+impl ModernPool {
+    pub(crate) const SIZE: usize = 0xa0;
+
+    pub(crate) fn encode(&self, output: &mut [u8]) -> Result<(), Error> {
+        let entry_bytes = (self.capacity as usize).checked_mul(8).ok_or(Error::Size)?;
+        let group_bytes = (self.populated as usize / 512).checked_mul(8).ok_or(Error::Size)?;
+        if !(1..=64).contains(&self.slot) || self.capacity == 0 || self.populated >= self.capacity || self.populated % 512 != 0
+            || self.entries.base() & 0x3fff != 0 || self.groups.base() & 0x3fff != 0
+            || self.entries.size() < entry_bytes || self.groups.size() < group_bytes
+            || self.submitted.get() & 7 != 0 || self.identity.get() & 7 != 0 { return Err(Error::Address); }
+        let group_count: u32 = group_bytes.try_into().map_err(|_| Error::Size)?;
+        let bytes = output.get_mut(..Self::SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        let word = |bytes: &mut [u8], offset: usize, value: u32| {
+            bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
+        };
+        bytes[..8].copy_from_slice(&self.identity.get().to_le_bytes());
+        word(bytes, 8, self.slot.into());
+        word(bytes, 12, (self.populated == 0).into());
+        word(bytes, 16, 2);
+        bytes[0x14..0x1c].copy_from_slice(&self.entries.base().to_le_bytes());
+        word(bytes, 0x1c, self.capacity);
+        word(bytes, 0x24, self.populated);
+        word(bytes, 0x28, (self.populated == 0).into());
+        word(bytes, 0x2c, self.populated);
+        bytes[0x30..0x38].copy_from_slice(&self.groups.base().to_le_bytes());
+        word(bytes, 0x40, 4);
+        word(bytes, 0x48, group_count);
+        bytes[0x4c..0x54].copy_from_slice(&self.submitted.get().to_le_bytes());
+        word(bytes, 0x5c, (self.compute_only && self.populated != 0).into());
+        word(bytes, 0x84, (self.populated != 0).into());
+        word(bytes, 0x94, 2);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod modern_tests {
+    use super::*;
+    fn pool(compute_only: bool) -> ModernPool {
+        ModernPool { identity: FirmwareVa::new(0xffff_fc21_9100_0000).unwrap(), slot: 1,
+            entries: GpuRegion::new(0x5000_0000, 0x20000).unwrap(),
+            groups: GpuRegion::new(0x6000_0000, 0x4000).unwrap(),
+            submitted: FirmwareVa::new(0xffff_fc21_9000_0000).unwrap(),
+            capacity: 0x4000, populated: 512, compute_only }
+    }
+    fn word(bytes: &[u8], offset: usize) -> u32 {
+        u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap())
+    }
+    #[test] fn shared_render_pool_never_gets_compute_exclusion() {
+        for compute in [false, true] {
+            let mut output = [0xa5; ModernPool::SIZE + 8];
+            pool(compute).encode(&mut output).unwrap();
+            assert_eq!(word(&output, 0x5c), compute.into());
+            assert_eq!(word(&output, 0x48), 8);
+            assert_eq!(word(&output, 0x84), 1);
+            assert_eq!(&output[ModernPool::SIZE..], &[0xa5; 8]);
+        }
+    }
+    #[test] fn bad_population_and_insufficient_owners_reject_before_mutation() {
+        for pages in [1, 511, 513, 0x4000] {
+            let mut value = pool(true); value.populated = pages;
+            let mut output = [0xa5; ModernPool::SIZE];
+            assert!(value.encode(&mut output).is_err());
+            assert_eq!(output, [0xa5; ModernPool::SIZE]);
+        }
+        let mut value = pool(true); value.capacity = 0x4001;
+        let mut output = [0xa5; ModernPool::SIZE];
+        assert!(value.encode(&mut output).is_err());
+        assert_eq!(output, [0xa5; ModernPool::SIZE]);
+    }
+}
+
+/// Populate the extended freelist from independently retained 4-MiB owners.
+/// Hardware group records describe 2-MiB subregions, not entire owners.
+pub(crate) struct ModernBacking<'a>(pub(crate) &'a [GpuRegion]);
+impl ModernBacking<'_> {
+    pub(crate) fn pages(&self) -> Result<usize, Error> {
+        if self.0.is_empty() || self.0.len() > 8
+            || self.0.iter().any(|owner| owner.size() != BACKING_SIZE || owner.base() & 0x3fff != 0) {
+            return Err(Error::Size);
+        }
+        Ok(self.0.len() * BACKING_SIZE / PAGE_SIZE)
+    }
+    pub(crate) fn page(&self, index: usize) -> Result<u64, Error> {
+        if index >= self.pages()? { return Err(Error::Index); }
+        let per_owner = BACKING_SIZE / PAGE_SIZE;
+        Ok(self.0[index / per_owner].base() + (index % per_owner * PAGE_SIZE) as u64)
+    }
+    pub(crate) fn group(&self, index: usize) -> Result<[u8; 64], Error> {
+        if index >= self.pages()? / 512 { return Err(Error::Index); }
+        let base = self.page(index * 512)?;
+        let mut record = [0; 64];
+        record[..8].copy_from_slice(&(base | (1 << 61)).to_le_bytes());
+        Ok(record)
+    }
+}
+
+#[cfg(test)]
+mod backing_tests {
+    use super::*;
+    #[test] fn group_records_never_cross_a_noncontiguous_backing_owner() {
+        let owners = [GpuRegion::new(0x5000_0000, BACKING_SIZE).unwrap(),
+            GpuRegion::new(0x6000_0000, BACKING_SIZE).unwrap()];
+        let backing = ModernBacking(&owners);
+        assert_eq!(backing.pages(), Ok(2048));
+        assert_eq!(backing.page(1024), Ok(0x6000_0000));
+        let second = backing.group(2).unwrap();
+        assert_eq!(u64::from_le_bytes(second[..8].try_into().unwrap()), (1 << 61) | 0x6000_0000);
+        assert_eq!(&second[8..], &[0; 56]);
+        assert!(backing.group(4).is_err());
+        assert!(backing.page(2048).is_err());
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_power_26.rs b/drivers/gpu/drm/asahi/m3/m3_power_26.rs
new file mode 100644
index 0000000000000..2619c24b4eb6a
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_power_26.rs
@@ -0,0 +1,414 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Build the extended M3 controller images from admitted device-tree inputs.
+//! These are constructor inputs. Firmware owns subsequent controller outputs.
+
+use super::{hw, initdata::PStateTables};
+use crate::{f32, float::F32};
+use kernel::prelude::*;
+
+struct Image(KVec<u8>);
+impl Image {
+    fn new(size: usize) -> Result<Self> {
+        let mut bytes = KVec::new();
+        bytes.resize(size, 0, GFP_KERNEL)?;
+        Ok(Self(bytes))
+    }
+    fn word(&mut self, offset: usize, value: u32) -> Result {
+        self.0
+            .get_mut(offset..offset.checked_add(4).ok_or(EOVERFLOW)?)
+            .ok_or(ERANGE)?
+            .copy_from_slice(&value.to_le_bytes());
+        Ok(())
+    }
+    fn pointer(&mut self, offset: usize, value: u64) -> Result {
+        self.0
+            .get_mut(offset..offset.checked_add(8).ok_or(EOVERFLOW)?)
+            .ok_or(ERANGE)?
+            .copy_from_slice(&value.to_le_bytes());
+        Ok(())
+    }
+    fn float(&mut self, offset: usize, value: F32) -> Result {
+        let bits = value.to_bits();
+        if bits & 0x7f80_0000 == 0x7f80_0000 {
+            return Err(EINVAL);
+        }
+        self.word(offset, bits)
+    }
+    fn reciprocal(&mut self, negative: usize, positive: usize, periods: u32) -> Result {
+        if periods == 0 {
+            return Err(EINVAL);
+        }
+        let alpha = f32!(1.0) / F32::from(periods);
+        self.float(negative, f32!(1.0) - alpha)?;
+        self.float(positive, alpha)
+    }
+}
+
+pub(crate) struct PowerImages {
+    pub(crate) hardware: KVec<u8>,
+    pub(crate) performance: KVec<u8>,
+    pub(crate) power: KVec<u8>,
+    pub(crate) globals: KVec<u8>,
+    pub(crate) maximum_khz: u32,
+}
+
+impl PowerImages {
+    /// `sram_uv` follows the same OPP iteration order as PwrConfig::perf_states.
+    /// No reference-bin voltage or experimental OPP-power estimate is substituted.
+    pub(crate) fn new(
+        pwr: &hw::PwrConfig,
+        sram_uv: &[u32],
+        sensor: u64,
+        retention: u64,
+        timestamp_base: u64,
+        io_records: &[u8],
+    ) -> Result<Self> {
+        let table = PStateTables::new(pwr, true, None, false)?;
+        let second = table.secondary.ok_or(EINVAL)?;
+        if table.len != 9
+            || table.max != 8
+            || sram_uv.len() != pwr.perf_states.len()
+            || pwr.perf_base_pstate == 0
+            || pwr.perf_base_pstate > 8
+            || !(1..=1000).contains(&pwr.power_sample_period)
+            || pwr.pwr_min_duty_cycle > 100
+            || pwr.avg_power_min_duty_cycle > 100
+            || io_records.len() != super::m3_mmio_26::SLOTS * super::m3_mmio_26::RECORD_BYTES
+            || [sensor, retention, timestamp_base]
+                .iter()
+                .any(|value| *value < 0xffff_fc00_0000_0000 || value & 7 != 0)
+        {
+            return Err(EINVAL);
+        }
+        let top = &pwr.perf_states[table.primary[8] as usize];
+        let base = &pwr.perf_states[table.primary[pwr.perf_base_pstate as usize] as usize];
+        if top.freq_hz != 1_338_000_000 || base.freq_hz >= top.freq_hz {
+            return Err(EINVAL);
+        }
+        let period = pwr.power_sample_period;
+        let ppm_periods = pwr.ppm_filter_time_constant_ms / period;
+        let average_periods = pwr.avg_power_filter_tc_ms / period;
+        if [
+            ppm_periods,
+            average_periods,
+            pwr.pwr_filter_time_constant,
+            pwr.perf_filter_time_constant,
+            pwr.perf_filter_time_constant2,
+            pwr.avg_power_target_filter_tc,
+            pwr.se_filter_time_constant,
+            pwr.se_filter_time_constant_1,
+        ]
+        .contains(&0)
+        {
+            return Err(EINVAL);
+        }
+        let clocks = |count: u32| {
+            count
+                .checked_mul(period)
+                .and_then(|v| v.checked_mul(24_000))
+                .ok_or(EOVERFLOW)
+        };
+        let dt = F32::from(period) / f32!(1000.0);
+        let ceiling = 800;
+        let base_units = pwr.perf_base_pstate * 100;
+        // The profile controller budget is independent of the loader's experimental
+        // OPP power model. It is not a demand floor or a replacement voltage table.
+        let budget = 22_044u32;
+        let mut main = Image::new(0x2658)?;
+        let mut performance = Image::new(0x6578)?;
+        let mut power = Image::new(0x670)?;
+        let mut globals = Image::new(0xe48)?;
+
+        main.pointer(0x28, timestamp_base)?;
+        main.0[0x640..0xe88].copy_from_slice(io_records);
+        main.pointer(0xe88, retention)?;
+        main.word(0xe90, 0x8122)?;
+        main.word(0xe94, 2)?;
+        main.word(0xe9c, 4)?;
+        main.word(0xea0, 1)?;
+        main.word(0xea4, 1)?;
+        main.word(0xeb8, 1)?;
+        main.word(0xec0, 1)?;
+        main.word(0xec8, 1)?;
+        main.word(0xed0, 24_000)?;
+        main.word(0xed4, 1)?;
+        main.word(0xed8, period)?;
+        main.word(0xee0, 1)?;
+        main.word(0xee4, 1)?;
+        main.word(0xee8, 1)?;
+        main.word(0xf04, 31)?;
+        main.pointer(0xf24, 4)?;
+        main.word(0xf34, 1)?;
+        // Firmware completion is polled through its normal bounded status path.
+        main.word(0xf38, 0)?;
+        main.word(0xf4c, 49)?;
+        main.word(0xf6c, 1)?;
+        main.word(0xf88, 10)?;
+        main.word(0xf8c, 3)?;
+        main.word(0xfac, 1)?;
+        main.word(0xfb8, 22)?;
+        main.word(0xfbc, 5)?;
+        main.word(0xfc0, 10)?;
+        main.word(0xfc4, 8)?;
+        main.word(0x1cd8, 8)?;
+        for state in 0..9 {
+            let primary = table.primary[state] as usize;
+            let secondary = second[state] as usize;
+            let item = &pwr.perf_states[primary];
+            let auxiliary = &pwr.perf_states[secondary];
+            if item.volt_mv.len() != 1
+                || auxiliary.volt_mv.len() != 1
+                || !(100..=1100).contains(&item.volt_mv[0])
+                || !(100..=1100).contains(&auxiliary.volt_mv[0])
+                || !(100_000..=1_100_000).contains(&sram_uv[primary])
+                || !(100_000..=1_100_000).contains(&sram_uv[secondary])
+                || sram_uv[primary].div_ceil(1000) < item.volt_mv[0]
+                || sram_uv[secondary].div_ceil(1000) < auxiliary.volt_mv[0]
+            {
+                return Err(EINVAL);
+            }
+            main.word(0xfc8 + state * 4, item.freq_hz / 1_000_000)?;
+            main.word(0x1808 + state * 4, auxiliary.freq_hz / 1_000_000)?;
+            main.float(0x1848 + state * 4, f32!(1.02))?;
+            for lane in 0..16 {
+                main.word(0x1008 + state * 64 + lane * 4, item.volt_mv[0])?;
+                main.word(
+                    0x1408 + state * 64 + lane * 4,
+                    sram_uv[primary].div_ceil(1000),
+                )?;
+            }
+            main.word(0x1cdc + state * 4, auxiliary.freq_hz / 1_000_000)?;
+            if state != 0 {
+                main.word(0x1d1c + state * 64, auxiliary.volt_mv[0])?;
+            }
+            main.word(0x211c + state * 64, sram_uv[secondary].div_ceil(1000))?;
+            if state != 0 {
+                // Profile indices for the admitted eight-state ladder.
+                let core_profile = if state <= 3 { state } else { 2 * state - 3 };
+                let auxiliary_profile = if state <= 4 { state } else { 2 * state - 4 };
+                main.word(0x19c8 + state * 4, core_profile as u32)?;
+                main.word(0x1a08 + state * 4, auxiliary_profile as u32)?;
+                let boost = item.freq_hz.saturating_sub(base.freq_hz) / 1_000_000;
+                main.word(
+                    0x1908 + state * 4,
+                    100 * boost / ((top.freq_hz - base.freq_hz) / 1_000_000),
+                )?;
+            }
+        }
+        // The native relative-power calibration of each running firmware state.
+        // Encode only these semantic percentages, not an object-image overlay.
+        for (state, percent) in [15, 25, 36, 46, 58, 75, 90, 100].into_iter().enumerate() {
+            main.word(0x18c8 + (state + 1) * 4, percent)?;
+        }
+        main.word(0x2544, 4)?;
+        main.word(0x2548, 6)?;
+        main.word(0x254c, 3)?;
+        main.word(0x2550, 7)?;
+        main.word(0x2554, 7)?;
+        main.word(0x2568, 10)?;
+        main.word(0x2570, 1)?;
+        main.word(0x258c, 0x160000)?;
+        main.word(0x2594, 1)?;
+        main.word(0x2598, 1)?;
+        main.word(0x259c, 1)?;
+        main.word(0x25a4, 1)?;
+        main.word(0x25a8, 1)?;
+        main.word(0x25ac, 1)?;
+        main.0[0x25b4..0x25e4].fill(0xff);
+        main.word(0x2604, 1)?;
+        main.word(0x2634, 1)?;
+
+        globals.word(0x24, 3000)?;
+        globals.word(0x78, 1)?;
+        globals.word(0x84, budget)?;
+        for offset in [0x88, 0x8c, 0x98] {
+            globals.word(offset, ceiling)?;
+        }
+        globals.word(0x998, 40)?;
+        globals.word(0x99c, 10)?;
+        globals.word(0x9a0, 250)?;
+        globals.word(0x9b8, 2)?;
+        globals.word(0x9bc, 40)?;
+        globals.word(0x9c0, 5)?;
+        globals.word(0x9c8, 40)?;
+        globals.word(0x9cc, 50)?;
+        globals.word(0xe28, 3)?;
+
+        power.word(0x10, average_periods)?;
+        power.float(0x14, pwr.avg_power_ki_only * dt)?;
+        power.float(0x18, pwr.avg_power_kp)?;
+        power.word(0x1c, pwr.avg_power_min_duty_cycle)?;
+        power.word(0x20, pwr.avg_power_target_filter_tc)?;
+        power.word(0xc8, 11_000)?;
+        power.word(0xcc, 8_000)?;
+        power.float(0xd8, pwr.fast_die0_proportional_gain)?;
+        power.float(0xdc, pwr.fast_die0_integral_gain * dt)?;
+        power.word(0x12c, budget)?;
+        power.float(0x130, pwr.ppm_kp)?;
+        power.float(0x134, pwr.ppm_ki * dt)?;
+        power.word(0x14c, 1)?;
+
+        performance.word(4, pwr.pwr_sample_period_aic_clks)?;
+        performance.word(8, pwr.pwr_sample_period_aic_clks)?;
+        performance.word(0x10, 4)?;
+        performance.float(0x14, f32!(1.0))?;
+        performance.word(0x2c, 1)?;
+        performance.word(0x30, 1)?;
+        // Leave the observed hardware state at zero until firmware writes it.
+        for offset in [0x40, 0x80, 0xb78, 0xb80, 0xbc0] {
+            performance.word(offset, base_units)?;
+        }
+        for offset in [0x44, 0x6c, 0x70, 0x84] {
+            performance.word(offset, 1)?;
+        }
+        for offset in [
+            0x48, 0x88, 0x9f4, 0x9f8, 0xa38, 0xaac, 0xaf0, 0xb7c, 0x221c, 0x22b4, 0x22b8, 0x22f8,
+            0x2870, 0x29f8, 0x29fc, 0x2a3c, 0x2b88, 0x2c44, 0x2c88,
+        ] {
+            performance.word(offset, ceiling)?;
+        }
+        performance.word(0x50, 100)?;
+        performance.word(0x60, 4)?;
+        performance.float(0x64, f32!(1.0))?;
+        performance.word(0x90, 100)?;
+        performance.word(0x9b8, 625)?;
+        performance.reciprocal(0x9c4, 0x9cc, pwr.pwr_filter_time_constant)?;
+        performance.float(0x9d4, pwr.pwr_integral_gain)?;
+        performance.float(0x9dc, F32::from(pwr.pwr_integral_min_clamp))?;
+        performance.float(0x9e0, F32::from(budget))?;
+        performance.float(0x9e4, pwr.pwr_proportional_gain)?;
+        performance.float(0x9ec, -F32::from(ceiling) / F32::from(budget))?;
+        for offset in [0x9f0, 0xaa8, 0x22b0] {
+            performance.word(offset, pwr.pwr_min_duty_cycle)?;
+        }
+        for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] {
+            performance.word(offset, budget)?;
+        }
+        performance.word(0xa70, ppm_periods.checked_mul(4).ok_or(EOVERFLOW)?)?;
+        performance.reciprocal(0xa7c, 0xa84, ppm_periods)?;
+        performance.float(0xa8c, pwr.ppm_ki * dt)?;
+        performance.float(0xa98, f32!(65536.0))?;
+        performance.float(0xa9c, pwr.ppm_kp)?;
+        performance.word(0xac8, pwr.ppm_filter_time_constant_ms)?;
+        performance.word(0xad0, clocks(ppm_periods)?)?;
+        performance.word(0xb28, pwr.perf_tgt_utilization)?;
+        performance.word(0xb30, pwr.perf_boost_min_util)?;
+        performance.word(0xb34, pwr.perf_boost_ce_step)?;
+        performance.word(0xb38, pwr.perf_reset_iters)?;
+        performance.word(0xb40, 6)?;
+        performance.word(0xb44, 1)?;
+        performance.word(0xb48, pwr.perf_filter_drop_threshold)?;
+        performance.reciprocal(0xb4c, 0xb54, pwr.perf_filter_time_constant)?;
+        performance.reciprocal(0xb50, 0xb58, pwr.perf_filter_time_constant2)?;
+        performance.float(0xb5c, pwr.perf_integral_gain)?;
+        performance.float(0xb60, pwr.perf_integral_gain2)?;
+        performance.float(0xb64, F32::from(pwr.perf_integral_min_clamp))?;
+        performance.float(0xb68, f32!(95.0))?;
+        performance.float(0xb6c, pwr.perf_proportional_gain)?;
+        performance.float(0xb70, pwr.perf_proportional_gain2)?;
+        performance.float(0xb74, F32::from(8 - pwr.perf_base_pstate) / f32!(0.95))?;
+        performance.word(0xb8c, pwr.perf_tgt_utilization)?;
+        performance.float(0x21f0, f32!(65536.0))?;
+        performance.float(0x21f4, F32::from(pwr.pwr_min_duty_cycle))?;
+        performance.float(0x21f8, F32::from(ceiling))?;
+        performance.word(0x21fc, 1338)?;
+        performance.float(0x2218, f32!(100.0))?;
+        performance.float(0x2224, f32!(0.8))?;
+        performance.float(0x2228, f32!(0.2))?;
+        performance.word(0x2268, 0x4248)?;
+        performance.word(0x2270, 8_000)?;
+        performance.word(0x2278, 4)?;
+        performance.float(0x228c, f32!(1.0))?;
+        performance.float(0x2294, pwr.fast_die0_integral_gain * dt)?;
+        performance.float(0x22a0, f32!(65536.0))?;
+        performance.float(0x22a4, pwr.fast_die0_proportional_gain)?;
+        performance.word(0x22c4, 11_000)?;
+        performance.word(0x28c0, 1)?;
+        performance.reciprocal(0x28dc, 0x28e0, pwr.avg_power_target_filter_tc)?;
+        performance.word(
+            0x28e4,
+            pwr.avg_power_target_filter_tc
+                .checked_mul(4)
+                .ok_or(EOVERFLOW)?,
+        )?;
+        performance.word(
+            0x28e8,
+            pwr.avg_power_target_filter_tc
+                .checked_mul(period)
+                .ok_or(EOVERFLOW)?,
+        )?;
+        performance.word(0x28ec, clocks(pwr.avg_power_target_filter_tc)?)?;
+        performance.word(0x29bc, average_periods.checked_mul(4).ok_or(EOVERFLOW)?)?;
+        performance.reciprocal(0x29c8, 0x29d0, average_periods)?;
+        performance.float(0x29d8, pwr.avg_power_ki_only * dt)?;
+        performance.float(0x29e4, f32!(65536.0))?;
+        performance.float(0x29e8, pwr.avg_power_kp)?;
+        performance.word(0x29f4, pwr.avg_power_min_duty_cycle)?;
+        performance.float(0x2a04, F32::from(budget))?;
+        performance.word(0x2a14, pwr.avg_power_filter_tc_ms)?;
+        performance.word(0x2a1c, clocks(average_periods)?)?;
+        performance.word(0x2b4c, 50)?;
+        performance.word(0x2b50, 1)?;
+        performance.reciprocal(0x2b58, 0x2b60, pwr.se_filter_time_constant)?;
+        performance.reciprocal(0x2b5c, 0x2b64, pwr.se_filter_time_constant_1)?;
+        performance.float(0x2b68, pwr.se_ki * dt)?;
+        performance.float(0x2b6c, pwr.se_ki_1 * dt)?;
+        performance.float(0x2b74, f32!(65536.0))?;
+        performance.float(0x2b78, pwr.se_kp)?;
+        performance.float(0x2b7c, pwr.se_kp_1)?;
+        performance.word(0x2b84, 100)?;
+        performance.word(0x2b8c, 100)?;
+        performance.word(0x2b98, pwr.se_target)?;
+        performance.word(
+            0x2ba4,
+            pwr.se_filter_time_constant
+                .checked_mul(period)
+                .ok_or(EOVERFLOW)?,
+        )?;
+        performance.word(
+            0x2ba8,
+            pwr.se_filter_time_constant_1
+                .checked_mul(period)
+                .ok_or(EOVERFLOW)?,
+        )?;
+        performance.word(0x2bac, clocks(pwr.se_filter_time_constant)?)?;
+        performance.word(0x2bb4, clocks(pwr.se_filter_time_constant_1)?)?;
+        performance.float(0x2b94, f32!(8000.0))?;
+        for offset in [0x2bc0, 0x2bc4, 0x2c30] {
+            performance.float(offset, f32!(65536.0))?;
+        }
+        performance.word(0x2be8, 8000)?;
+        performance.word(0x2bec, 4)?;
+        performance.word(0x2bf0, pwr.se_inactive_threshold)?;
+        performance.word(0x2bf4, pwr.se_engagement_criteria as u32)?;
+        performance.word(0x2bf8, 2)?;
+        performance.word(0x2bfc, 4)?;
+        performance.word(0x2c00, pwr.se_reset_criteria)?;
+        performance.word(0x2c40, pwr.avg_power_min_duty_cycle)?;
+        performance.word(0x2e2c, 4)?;
+        performance.float(0x2e38, f32!(0.96875))?;
+        performance.float(0x2e40, f32!(0.03125))?;
+        performance.float(0x2e48, f32!(5.0))?;
+        performance.float(0x2e54, f32!(65536.0))?;
+        performance.float(0x2e58, f32!(15.0))?;
+        performance.word(0x2e64, 700)?;
+        for offset in [0x2e68, 0x2e6c, 0x2e78] {
+            performance.word(offset, ceiling)?;
+        }
+        performance.word(0x2f08, 0x4248)?;
+        performance.word(0x2f10, 125)?;
+        performance.word(0x2f14, 1)?;
+        performance.pointer(0x3b50, sensor)?;
+        performance.word(0x3b58, 0x4b48)?;
+        performance.word(0x3b60, 1)?;
+        performance.word(0x62a8, 1)?;
+        Ok(Self {
+            hardware: main.0,
+            performance: performance.0,
+            power: power.0,
+            globals: globals.0,
+            maximum_khz: top.freq_hz / 1000,
+        })
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs
new file mode 100644
index 0000000000000..04b872ed9ea80
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Firmware-interface properties, independent of the physical G15 GPU record format.
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Profile {
+    Legacy,
+    Extended,
+}
+
+pub(crate) const PARAMS_REQUIRED: u64 = 600;
+
+impl Profile {
+    pub(crate) const fn firmware_compat(self) -> [u32; 3] {
+        match self {
+            Self::Legacy => [14, 8, 3],
+            Self::Extended => [26, 6, 2],
+        }
+    }
+
+    pub(crate) const fn hal(self) -> u32 {
+        match self {
+            Self::Legacy => 0,
+            Self::Extended => 200,
+        }
+    }
+
+    pub(crate) const fn user_end(self) -> u64 {
+        match self {
+            Self::Legacy => 0x2ff_ffff_8000,
+            Self::Extended => (1 << 42) - 2 * 0x4000,
+        }
+    }
+
+    /// Keep the diagnostic AP/GPU timeline in its established 24 MHz units.
+    /// User timestamps and duration calculations retain native firmware units.
+    pub(crate) const fn profiler_ticks(self, timestamp: u64) -> u64 {
+        match self {
+            Self::Legacy => timestamp,
+            Self::Extended => (timestamp / 1000) * 24 + (timestamp % 1000) * 24 / 1000,
+        }
+    }
+
+    pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option<u64> {
+        match self {
+            Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24),
+            Self::Extended => Some(elapsed),
+        }
+    }
+
+    pub(crate) const fn vm_acknowledged(self, params_bytes: u64, flags: u32) -> bool {
+        // Both profiles consume G15 records; the existing VM padding stays zero.
+        params_bytes >= PARAMS_REQUIRED && flags == 0
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn profiling_uses_one_clock_domain_without_changing_native_durations() {
+        for value in [0, 1, 999, 1000, 24_000_000, u64::MAX] {
+            assert_eq!(Profile::Legacy.profiler_ticks(value), value);
+            assert_eq!(
+                Profile::Extended.profiler_ticks(value),
+                ((value as u128) * 24 / 1000) as u64
+            );
+        }
+        assert_eq!(Profile::Extended.profiler_ticks(1_000_000_000), 24_000_000);
+    }
+
+    #[test]
+    fn profile_acknowledgement_is_exact_and_file_query_covers_hal() {
+        for profile in [Profile::Legacy, Profile::Extended] {
+            let expected = 0;
+            for size in [0, 599, 600, 616, u64::MAX] {
+                for flags in [0, 1, 2, 3, u32::MAX] {
+                    assert_eq!(
+                        profile.vm_acknowledged(size, flags),
+                        size >= 600 && flags == expected
+                    );
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn firmware_interface_version_does_not_change_physical_identity() {
+        assert_eq!(Profile::Legacy.hal(), 0);
+        assert_eq!(Profile::Extended.hal(), 200);
+        assert_eq!(Profile::Legacy.firmware_compat(), [14, 8, 3]);
+        assert_eq!(Profile::Extended.firmware_compat(), [26, 6, 2]);
+        assert_eq!(Profile::Extended.user_end(), 0x3ff_ffff_8000);
+        assert_eq!(Profile::Legacy.user_end(), 0x2ff_ffff_8000);
+    }
+    #[test]
+    fn native_nanoseconds_are_never_rescaled_as_legacy_ticks() {
+        assert_eq!(
+            Profile::Extended.gpu_duration_ns(24_000_000),
+            Some(24_000_000)
+        );
+        assert_eq!(
+            Profile::Legacy.gpu_duration_ns(24_000_000),
+            Some(1_000_000_000)
+        );
+        assert_eq!(Profile::Extended.gpu_duration_ns(u64::MAX), Some(u64::MAX));
+        assert_eq!(Profile::Legacy.gpu_duration_ns(u64::MAX), None);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs
index 3daba7acba2d0..8fd5ffdf1d30d 100644
--- a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs
@@ -113,3 +113,44 @@ pub(crate) fn ring_entry(index: u32, capacity: u32, seeds: &[FirmwareVa]) -> Res
     if seeds.is_empty() || seeds.len() > capacity as usize || capacity > 65536 || index >= capacity { return Err(Error::Index); }
     Ok(if (index as usize) < seeds.len() { seeds[index as usize].0.to_le_bytes() } else { [0xcc; 8] })
 }
+
+impl QueueInfo {
+    pub(crate) const EXTENDED_SIZE: usize = 0x24c0;
+
+    /// Scheduler storage belongs to this same retained allocation; it is not
+    /// interchangeable with the legacy GPU scratch object.
+    pub(crate) fn encode_extended(&self, owner: FirmwareVa, output: &mut [u8]) -> Result<(), Error> {
+        let end = owner.get().checked_add(Self::EXTENDED_SIZE as u64).ok_or(Error::Address)?;
+        if end > 0xffff_fc30_0000_0000 { return Err(Error::Address); }
+        let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        for (offset, value) in [(0, self.state.get()), (8, self.ring.get()),
+            (16, self.job_list.get()), (24, owner.get() + 0xb0), (0xa4, self.context.get())] {
+            bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
+        }
+        for offset in [0x2c, 0x4c] { bytes[offset..offset + 4].fill(0xff); }
+        for offset in [0x30, 0x34, 0x48] {
+            bytes[offset..offset + 4].copy_from_slice(&2u32.to_le_bytes());
+        }
+        bytes[0x38..0x40].copy_from_slice(&0xffff_0000_0000_0000u64.to_le_bytes());
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    #[test] fn scheduler_pointer_relocates_with_its_retained_queue_owner() {
+        let fw = |i: u64| FirmwareVa::new(0xffff_fc21_4000_0000 + i * 0x4000).unwrap();
+        let info = QueueInfo { state: fw(1), ring: fw(2), job_list: fw(3),
+            gpu_scratch: fw(4), context: fw(5), uuid: 9 };
+        let mut output = vec![0xa5; QueueInfo::EXTENDED_SIZE + 8];
+        info.encode_extended(fw(0), &mut output).unwrap();
+        assert_eq!(u64::from_le_bytes(output[24..32].try_into().unwrap()), fw(0).get() + 0xb0);
+        assert_eq!(&output[0xb0..QueueInfo::EXTENDED_SIZE], &vec![0; QueueInfo::EXTENDED_SIZE - 0xb0]);
+        assert_eq!(&output[QueueInfo::EXTENDED_SIZE..], &[0xa5; 8]);
+        let mut short = [0xa5; INFO_SIZE];
+        assert!(info.encode_extended(fw(0), &mut short).is_err());
+        assert_eq!(short, [0xa5; INFO_SIZE]);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs
new file mode 100644
index 0000000000000..9a9f63d15b784
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_render.rs
@@ -0,0 +1,807 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+use kernel::prelude::*;
+use crate::m3::m3_queue_layout as queue;
+use crate::m3::m3_sync_layout as sync;
+use crate::m3::m3_state_layout as state;
+use crate::m3::m3_scene_layout as scene;
+use crate::{m3::m3_pass::{Pass,ViewOwner},m3::m3_pass_layout::{self as pass_layout,Field as P,Space}};
+use crate::{m3::m3_fragment_command as fragment, m3::m3_tiler_command as tiler, m3::m3_render_sequence as sequence, m3::m3_init_layout::Region};
+use crate::{m3::m3_parameter_layout as parameter, m3::m3_pool_layout::GpuRegion};
+use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_shared_layout as s,m3::agx_uapi};
+use crate::m3::m3_timeline as timeline;
+use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE};
+const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32;
+pub(crate) const PB_PAGES:u32=PB_BLOCKS*4;
+// Serial execution permits retaining each client's views of the owned arenas.
+// Bound the cache to four VMs including the active one; cache entries retain
+// their ASID lease and all aliases until eviction after exact retirement.
+struct CachedViews {
+    aliases:KVec<mmu::KernelMapping>,
+    views:KVec<(ViewOwner,mmu::KernelMapping)>,
+    // Release mappings before returning their ASID lease (Rust field order).
+    binding:mmu::VmBind,
+}
+pub(crate) struct Render {
+    _aliases:KVec<mmu::KernelMapping>, objects:KVec<Buffer>,
+    passes:KVec<Pass>, init_bm:Buffer,
+    // All active mappings above must drop before their ASID binding.
+    _binding:mmu::VmBind,
+    stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec<u8>, tiler_bytes: KVec<u8>, fragment_bytes: KVec<u8>,
+    last_progress_ns:i64,
+    draw:u64, heads:[u16;2], cached_views:KVec<CachedViews>,
+    slot:usize, batch_count:usize, early_count:u32,
+    /// Measurement packets in the pass most recently appended to its queues.
+    stage_timestamps:[bool;2],
+    /// First pass slot of the current batch; its passes use base..base+batch_count mod SLOTS.
+    base:usize,
+    checkpoint:(u64,[u16;2],usize,usize,u32,usize),
+    /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers.
+    clusters:u32,
+    /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`).
+    registers:crate::m3::m3_compute_layout::RegisterSet,
+    /// Last: drops after the pass aliases.
+    _lower_tlbi:crate::m3::m3_memory::LowerTlbi,
+}
+// Queue allocations belong to Render, not to a pass. Keep their fixed arenas
+// and firmware/shared mapping permissions from the qualified allocation map.
+// Scratch is GPU-visible firmware storage; it is not the application's VM.
+struct QueueObjects {
+    info: usize, state: usize, ring: usize, scratch: usize,
+    tiler: bool, uuid: u32,
+}
+use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST,
+    PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC};
+const MANAGER_SCENE_COUNT: u32 = 48;
+const TA_POOL_COUNTER: usize = 11;
+const UMA_POOLS: [crate::m3::m3_pool::Owners; 2] = [
+    crate::m3::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 },
+    crate::m3::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 },
+];
+const FRAGMENT_POOL_COUNTER: usize = 23;
+// Retained producer identifier for cross-pass dependencies, G15 / V14_8_3.
+const DEPENDENCY_UUID: u32 = 250912;
+const RENDER_QUEUES: [QueueObjects; 2] = [
+    QueueObjects { info: s::TA_QUEUE, state: s::TA_POINTERS, ring: s::TA_RING,
+        scratch: s::TA_SCRATCH, tiler: true, uuid: 0x7a0000 },
+    QueueObjects { info: s::FRAGMENT_QUEUE, state: s::FRAGMENT_POINTERS, ring: s::FRAGMENT_RING,
+        scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 },
+];
+impl Render {
+    pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:[Region;2],clusters:u32,
+        registers:crate::m3::m3_compute_layout::RegisterSet,profile:super::m3_profile::Profile)->Result<Self> {
+        let lower_tlbi=crate::m3::m3_memory::LowerTlbi;
+        let binding=uat.bind(vm)?;
+        let mut objects=KVec::new();let mut aliases=KVec::new();
+        for index in 0..s::COUNT {
+            let a=s::allocation_for(profile,index).map_err(|_|EINVAL)?;
+            let protection=prot::Prot::from_pte(a.access.pte());
+            let b=match a.space {
+                Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None,
+                    a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?,
+                Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),
+                    a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address),
+                    a.size,protection,prot::PROT_GPU_SHARED_RW)?,
+            };
+            let mut b=b;
+            if a.gpu_alias.is_some() { aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?; }
+            objects.push(b,GFP_KERNEL)?;
+        }
+        // InitBM is shared, not part of a pass. Preserve its qualified mapping.
+        let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(if profile==super::m3_profile::Profile::Extended {0xffff_fc24_4000_0000}
+                else {0xfffffc200062bfe0}),
+            sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?;
+        let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?;
+        for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,profile,&mut aliases)?,GFP_KERNEL)?;}
+        let mut sequence_bytes=KVec::new();
+        sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?;
+        let mut tiler_bytes=KVec::new();
+        tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?;
+        let mut fragment_bytes=KVec::new();
+        fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?;
+        let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi};
+        // Lists and the matching manager are built before InitBM publication.
+        job.initialize_parameter_buffer()?;
+        for (index,pool) in UMA_POOLS.iter().enumerate() {
+            if profile==super::m3_profile::Profile::Extended {
+                pool.initialize_extended(&mut job.objects,33+index as u8,false)?;
+            } else {pool.initialize(&mut job.objects)?;}
+        }
+        job.initialize_state()?;
+        job.initialize_queues()?;
+        dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024));
+        crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();
+        unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))};
+        Ok(job)
+    }
+    fn initialize_parameter_buffer(&mut self) -> Result {
+        let region = |owner: usize| GpuRegion::new(self.objects[owner].gpu_va()?, self.objects[owner].size())
+            .map_err(|_| EINVAL);
+        let pages = region(PARAMETER_PAGE_LIST)?;
+        let blocks = region(PARAMETER_BLOCK_LIST)?;
+        let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?;
+        let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?;
+        let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?;
+        let extended=self.profile==super::m3_profile::Profile::Extended;
+        let mut bytes = [0; parameter::Manager::EXTENDED_SIZE];
+        let length=if extended {
+            manager.encode_extended(queue::FirmwareVa::new(self.objects[PARAMETER_PAGE_LIST].va()).map_err(|_|EINVAL)?,
+                queue::FirmwareVa::new(self.objects[PARAMETER_BLOCK_LIST].va()).map_err(|_|EINVAL)?,0,&mut bytes)
+                .map_err(|_|EINVAL)?;
+            parameter::Manager::EXTENDED_SIZE
+        } else {manager.encode(&mut bytes).map_err(|_|EINVAL)?;parameter::MANAGER_SIZE};
+        for group in 0..PB_GROUPS {
+            let buffer = &self.objects[PB_FIRST + group];
+            let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?)
+                .map_err(|_| EINVAL)?;
+            for block in 0..PB_BLOCKS_PER_GROUP {
+                self.objects[PARAMETER_BLOCK_LIST].write((group * PB_BLOCKS_PER_GROUP + block) * 8,
+                    &backing.block(block).map_err(|_| EINVAL)?)?;
+            }
+            for page in 0..PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK {
+                self.objects[PARAMETER_PAGE_LIST].u32((group * PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK + page) * 4,
+                    backing.page(page).map_err(|_| EINVAL)?)?;
+            }
+        }
+        self.objects[BUFFER_MANAGER].write(0, &bytes[..length])
+    }
+    fn initialize_state(&mut self) -> Result {
+        let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL);
+        let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() };
+        let list_owner = address(JOB_LIST)?;
+        let mut context = [0; state::CONTEXT_SIZE];
+        if self.profile==super::m3_profile::Profile::Extended {
+            state::context_extended(&mut context).map_err(|_|EINVAL)?;
+            self.objects[GPU_CONTEXT].write(0,&context[..state::EXTENDED_CONTEXT_SIZE])?;
+            for engine in 0..2 {
+                let value=state::Notifier {threshold:queue::FirmwareVa::new(self.objects[s::EVENT_COUNT].va()+engine as u64*8).map_err(|_|EINVAL)?,
+                    context:self._binding.slot()};
+                let mut notifier=[0;state::EXTENDED_NOTIFIER_SIZE];
+                value.encode_extended(&mut notifier).map_err(|_|EINVAL)?;
+                self.objects[NOTIFIER].write(engine*state::EXTENDED_NOTIFIER_SIZE,&notifier)?;
+            }
+        } else {
+            state::context(&mut context).map_err(|_| EINVAL)?;
+            self.objects[GPU_CONTEXT].write(0, &context)?;
+            let mut notifier = [0; state::NOTIFIER_SIZE];
+            notification.encode(&mut notifier).map_err(|_| EINVAL)?;
+            self.objects[NOTIFIER].write(0, &notifier)?;
+        }
+        let mut list = [0; state::JOB_LIST_SIZE];
+        state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?;
+        self.objects[JOB_LIST].write(0, &list)?;
+        let mut manager = [0; state::MANAGER_STATE_SIZE];
+        state::block_control(&mut manager, PB_BLOCKS, PB_BLOCKS).map_err(|_| EINVAL)?;
+        self.objects[BLOCK_CONTROL].write(0, &manager)?;
+        state::manager_misc(&mut manager).map_err(|_| EINVAL)?;
+        self.objects[MANAGER_MISC].write(0, &manager)?;
+        for (owner, value) in [(TA_POOL_COUNTER, 1), (FRAGMENT_POOL_COUNTER, 1),
+            (s::EVENT_COUNT, 2), (s::BM_COUNTER, 1),
+            (s::TA_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), (s::TA_FW_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)),
+            (s::FRAGMENT_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0)), (s::FRAGMENT_FW_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0))] {
+            let mut bytes = [0; state::MANAGER_STATE_SIZE];
+            let bytes = bytes.get_mut(..self.objects[owner].size()).ok_or(EINVAL)?;
+            state::counter(bytes, value).map_err(|_| EINVAL)?;
+            self.objects[owner].write(0, bytes)?;
+        }
+        if self.profile==super::m3_profile::Profile::Extended {
+            self.objects[s::EVENT_COUNT].fill(0)?;
+            self.objects[s::BM_COUNTER].fill(0)?;
+        }
+        Ok(())
+    }
+    fn initialize_queues(&mut self) -> Result {
+        for owner in &RENDER_QUEUES {
+            let address = |index: usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_| EINVAL);
+            let info = queue::QueueInfo {
+                state: address(owner.state)?, ring: address(owner.ring)?,
+                job_list: address(JOB_LIST)?, gpu_scratch: address(owner.scratch)?,
+                context: address(GPU_CONTEXT)?, uuid: owner.uuid,
+            };
+            let seed_addresses=if owner.tiler {
+                [self.init_bm.va(),self.passes[0].get(P::TilerCommand).va()]
+            }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]};
+            let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?,
+                queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?];
+            let mut bytes=KVec::new();
+            bytes.resize(if self.profile==super::m3_profile::Profile::Extended {
+                queue::QueueInfo::EXTENDED_SIZE
+            } else {queue::INFO_SIZE},0u8,GFP_KERNEL)?;
+            if self.profile==super::m3_profile::Profile::Extended {
+                info.encode_extended(address(owner.info)?,&mut bytes).map_err(|_|EINVAL)?;
+            } else {info.encode(&mut bytes).map_err(|_|EINVAL)?;}
+            self.objects[owner.info].write(0, &bytes)?;
+            let mut state = [0; queue::STATE_SIZE];
+            queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 }
+                .encode(&mut state).map_err(|_| EINVAL)?;
+            self.objects[owner.state].write(0, &state)?;
+            for index in 0..queue::QUALIFIED_CAPACITY {
+                let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &seeds).map_err(|_| EINVAL)?;
+                self.objects[owner.ring].write(index as usize * 8, &entry)?;
+            }
+        }
+        Ok(())
+    }
+    fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result {
+        let mut bytes=[0;sync::Barrier::EXTENDED_SIZE];
+        let length=if self.profile==super::m3_profile::Profile::Extended {
+            command.encode_extended(&mut bytes).map_err(|_|EINVAL)?;
+            sync::Barrier::EXTENDED_SIZE
+        } else {command.encode(&mut bytes).map_err(|_|EINVAL)?;sync::BARRIER_SIZE};
+        self.passes[self.slot].get_mut(owner).write(0,&bytes[..length])
+    }
+    // Construct synchronization from the actual owners after resetting a pass,
+    // before publishing its ring entries. InitBM is shared and dispatched only
+    // for draw one; rewriting its bytes after retirement preserves the old path.
+    fn prepare_sync(&mut self, early: bool, draw: u64,
+        predecessor:Option<sync::CompletionDependency>) -> Result {
+        let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL);
+        let manager = address(BUFFER_MANAGER)?;
+        let ta_stamp = address(s::TA_FW_STAMP)?;
+        let fs_stamp = address(s::FRAGMENT_FW_STAMP)?;
+        let mut init = [0; sync::INIT_BM_SIZE];
+        let value=sync::InitBufferManager { context: self._binding.slot(), slot: 0,
+            block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) };
+        if self.profile==super::m3_profile::Profile::Extended {
+            value.encode_extended(&mut init).map_err(|_|EINVAL)?;
+        } else {value.encode(&mut init).map_err(|_|EINVAL)?;}
+        self.init_bm.write(0, &init)?;
+        let ta_value = timeline::stamp(s::STAMP_TA - 256, draw);
+        let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw);
+        let previous=predecessor.unwrap_or(sync::CompletionDependency {
+            stamp:if early {ta_stamp} else {fs_stamp},
+            value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1),
+            event:u32::from(!early),
+        });
+        self.write_barrier(P::TilerToFragment, sync::Barrier {
+            stamp: ta_stamp, wait_value: ta_value, event: 0, self_value: fs_value,
+            uuid: 0x3d0000, dependency: sync::Dependency::TilerToFragment })?;
+        self.write_barrier(P::TilerDependency, sync::Barrier {
+            stamp: previous.stamp, wait_value: previous.value,
+            event: previous.event, self_value: ta_value,
+            uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?;
+        self.write_barrier(P::FragmentDependency, sync::Barrier {
+            stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value,
+            uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })
+    }
+    // Prepare each private command after its retired slot is reset. The
+    // reusable host staging allocation is copied before the next slot is built.
+    fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result {
+        use crate::m3::m3_compute_layout::GpuVa;
+        let pass=&self.passes[self.slot];
+        let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL);
+        let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL);
+        let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL);
+        let seq=pass.get(P::TilerSequence);
+        let command=pass.get(P::TilerCommand);
+        let g=Self::geometry(r,self.clusters)?;
+        if g.tpc_bytes>pass.get(P::Tpc).size() as u64
+            || g.tilemap_bytes>pass.get(P::Tilemap).size() as u64 {return Err(E2BIG);}
+        // The old per-slot register keeps its seed entry, independent of the
+        // draw-dependent RenderScene reference. Do not change this policy here.
+        let initial_scene_entry=queue::FirmwareVa::new(self.objects[s::BM_SCENES].va()
+            +(self.slot as u64+1)*4).map_err(|_|EINVAL)?;
+        let value=tiler::Command {
+            context:self._binding.slot(),register_set:self.registers,
+            notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,
+            scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,
+            sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?,
+            scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?,
+            stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?],
+            user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?],
+            tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?,
+            scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES,
+            vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?,
+            multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0,
+        };
+        if self.profile==super::m3_profile::Profile::Extended {
+            value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?,
+                peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc,
+                firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?,
+                scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?;
+        } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;}
+        self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes)
+    }
+    fn prepare_sequences(&mut self,draw:u64)->Result {
+        for fs in [false,true] {
+            let pass=&self.passes[self.slot];
+            let region=|field:P| {let b=pass.get(field);Region::new(b.va(),b.size()).map_err(|_|EINVAL)};
+            let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL);
+            let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL);
+            let target=if fs{P::FragmentSequence}else{P::TilerSequence};
+            let notifier=&self.objects[NOTIFIER];
+            let owners=sequence::Owners {
+                command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?,
+                stats:self.stats[usize::from(fs)],notifier:Region::new(notifier.va()+
+                    if fs && self.profile==super::m3_profile::Profile::Extended {0x200} else {0},
+                    if self.profile==super::m3_profile::Profile::Extended {0x200} else {notifier.size()}).map_err(|_|EINVAL)?,
+                queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?,
+                manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?,
+                fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?,
+                context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw),
+            };
+            let timed=self.stage_timestamps[usize::from(fs)];
+            let length=if self.profile==super::m3_profile::Profile::Extended {
+                let pool=&self.objects[UMA_POOLS[usize::from(fs)].manager];
+                let auxiliary=super::m3_compute_layout::GpuVa::new(pool.gpu_va()?+if fs{0x200}else{0x100}).map_err(|_|EINVAL)?;
+                let serial=draw.checked_mul(2).and_then(|v|v.checked_sub(u64::from(fs))).ok_or(EOVERFLOW)?;
+                owners.encode_extended(fs,auxiliary,serial,&mut self.sequence_bytes).map_err(|_|EINVAL)?
+            } else if fs {
+                let b=pass.get(P::FragmentScratch);
+                let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?;
+                sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)?
+            }else{sequence::Tiler{owners,scratch:fw(P::TilerScratch)?}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)?};
+            self.passes[self.slot].get_mut(target).write(0,&self.sequence_bytes)?;
+            self.passes[self.slot].get_mut(if fs{P::FragmentCommand}else{P::TilerCommand}).u32(sequence::command_length_offset(fs),length as u32)?;
+        }
+        Ok(())
+    }
+    fn view_owner(&mut self,owner:ViewOwner)->&mut Buffer {
+        match owner {ViewOwner::Shared(index)=>&mut self.objects[index],
+            ViewOwner::Pass{slot,field}=>self.passes[slot].get_mut(field)}
+    }
+    fn patch_context(&mut self)->Result {
+        let context=self._binding.slot();
+        self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?;
+        if self.profile==super::m3_profile::Profile::Extended {
+            self.objects[NOTIFIER].u32(0x200+state::NOTIFIER_CONTEXT,context)?;
+        }
+        Ok(())
+    }
+    /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster.
+    fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result<crate::m3::agx_render::Geometry> {
+        use crate::m3::agx_render::{Geometry,Utile};
+        let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32};
+        let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32};
+        let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?;
+        g.set_samples(r.samples).map_err(|_|EINVAL)?;
+        Ok(g)
+    }
+    fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result {
+        use crate::{m3::m3_compute_layout::GpuVa,m3::agx_render_state::{Program,DepthStencil}};
+        let pass=&self.passes[self.slot];
+        let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL);
+        let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL);
+        let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL);
+        let seq=pass.get(P::FragmentSequence);
+        let command=pass.get(P::FragmentCommand);
+        let geometry=Self::geometry(r,self.clusters)?;
+        let mask=if r.flags&16!=0 {u64::MAX} else {u32::MAX as u64};
+        let program=|p:agx_uapi::UapiProgram|->Result<Program> {Ok(Program {
+            address:usc.checked_add(u64::from(p.usc&!63)).ok_or(EOVERFLOW)?,resources:p.resource_spec&mask,
+        })};
+        let tilebuffer_control=(u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)).div_ceil(2048)
+            |(u32::from(r.samples.trailing_zeros() as u8)<<17);
+        let tile_mode=0x280|u64::from(r.layers>1)|if r.flags&2!=0 {1<<16}else{0};
+        let value=fragment::Command {
+            context:self._binding.slot(),register_set:self.registers,
+            sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,
+            notifier:queue::FirmwareVa::new(self.objects[NOTIFIER].va()+
+                if self.profile==super::m3_profile::Profile::Extended {0x200} else {0}).map_err(|_|EINVAL)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},
+            gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?,
+            heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?,
+            scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?,
+            fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw),
+            timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?],
+            state:fragment::State {
+                geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y],
+                eot:program(r.end_of_tile)?,background:program(r.background)?,
+                partial_eot:program(r.partial_end_of_tile)?,partial_background:program(r.partial_background)?,
+                scissor:r.scissor_base,depth_bias:r.depth_bias_base,query:r.occlusion_query_base,
+                isp_control:0xc000|(r.flags&(1<<18)),tilebuffer_control,mirror_tilebuffer_control:tilebuffer_control,
+                tile_mode,mirror_tile_mode:tile_mode as u32,
+                depth:DepthStencil{base:r.depth.base,stride:u64::from(r.depth.stride)},
+                stencil:DepthStencil{base:r.stencil.base,stride:u64::from(r.stencil.stride)},
+                depth_compression:DepthStencil{base:r.depth.compression_base,stride:u64::from(r.depth.compression_stride)},
+                stencil_compression:DepthStencil{base:r.stencil.compression_base,stride:u64::from(r.stencil.compression_stride)},
+                zls_control:r.zls_control,depth_dimensions:r.depth_dimensions,depth_clear:r.depth_clear,
+                stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0,
+            },
+        };
+        if self.profile==super::m3_profile::Profile::Extended {
+            value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).and_then(|v|v.checked_sub(1)).ok_or(EOVERFLOW)?,
+                peer_stamp:0,usc_window:usc,
+                firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[1].manager].va()+0x200).map_err(|_|EINVAL)?,
+                scene_scratch:gpu(P::FragmentScratch)?},&mut self.fragment_bytes).map_err(|_|EINVAL)?;
+        } else {value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?;}
+        self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes)
+    }
+    pub(crate) fn set_attachments(&mut self,attachments:&[crate::m3::agx_attachments::Attachments;2])->Result {
+        // Preserve the legacy renderer's existing packet format. The native
+        // interface consumes separate attachment apertures for both stages.
+        if self.profile!=super::m3_profile::Profile::Extended {return Ok(());}
+        for (fragment,list) in attachments.iter().enumerate() {
+            let field=if fragment==0 {P::TilerSequence}else{P::FragmentSequence};
+            let buffer=self.passes[self.slot].get_mut(field);
+            sequence::patch_extended_attachments(fragment!=0,list,|offset,data|buffer.write(offset,data))?;
+        }
+        crate::m3::agx_memory::publish();Ok(())
+    }
+    pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result {
+        if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);}
+        for (stage,(command,sequence,offset,start,end)) in [
+            (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END),
+            (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END)
+        ].into_iter().enumerate() {
+            let (offset,start,end)=if self.profile==super::m3_profile::Profile::Extended {
+                if stage==0 {(0x8d8,0x1cc,0x21c)} else {(0xc20,0x1e4,0x234)}
+            } else {(offset,start,end)};
+            let pair=addresses[stage];
+            if !self.stage_timestamps[stage] {continue;}
+            self.passes[self.slot].get_mut(command).u64(offset,pair[0])?;
+            self.passes[self.slot].get_mut(command).u64(offset+8,pair[1])?;
+            let pointer=if pair==[0,0] {0} else {self.passes[self.slot].get_mut(command).va()+offset as u64};
+            self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?;
+            self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?;
+        }
+        crate::m3::agx_memory::publish();Ok(())
+    }
+    fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS}
+    /// GPU time of the batch of `count` passes starting at pass slot `base`.
+    pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result<u64> {
+        let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?;
+        let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?;
+        self.profile.gpu_duration_ns(if start!=0 && end>=start {end-start} else {0}).ok_or(EOVERFLOW)
+    }
+    /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks.
+    pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> {
+        Ok((self.profile.profiler_ticks(self.passes[base].get_mut(P::TilerStart).read_u64(0)?),
+            self.profile.profiler_ticks(self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)))
+    }
+    /// Read retained timestamps only after exact batch retirement. The first
+    /// TA and final fragment delimit the batch in the shared 24 MHz counter.
+    pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> {
+        if self.batch_count == 0 {return Err(EINVAL);}
+        let (start,end)=self.gpu_ticks(self.base,self.batch_count)?;
+        Ok([start,end])
+    }
+    pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> {
+        let mut total=[0;3];
+        for index in 0..count {
+            let stages=self.slot_stage_ns(base,count,index)?;
+            for i in 0..3 {total[i]+=stages[i];}
+        }
+        Ok(total)
+    }
+    pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> {
+        if index>=count {return Err(EINVAL);}
+        let slot=Self::at(base,index);
+        let duration=|a:u64,b:u64| self.profile.gpu_duration_ns(if a!=0 && b>=a {b-a} else {0}).unwrap_or(0);
+        let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?;
+        let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?;
+        let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?;
+        let ta_end=self.passes[slot].get_mut(P::TilerEnd).read_u64(0)?;
+        Ok([duration(ta_start,ta_end),duration(fs_start,fs_end),duration(ta_end,fs_start)])
+    }
+    pub(crate) fn early_count(&self)->u32 {self.early_count}
+    pub(crate) fn heads(&self)->[u16;2] {self.heads}
+    pub(crate) fn ordinal(&self)->u64 {self.draw}
+    /// The current batch: first pass slot, pass count and last draw number.
+    pub(crate) fn batch(&self)->(usize,usize,u64) {(self.base,self.batch_count,self.draw)}
+    /// Whether draws 1..=`draw` have completed both stages (TA and fragment stamps, host and
+    /// firmware copies, have reached `draw`'s values). Use signed modular ordering
+    /// across stamp wrap; the runtime bounds outstanding passes to SLOTS.
+    pub(crate) fn complete_through(&mut self,draw:u64)->Result<bool> {
+        if draw==0 {return Ok(true);}
+        for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)),
+            (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] {
+            if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);}
+        }
+        Ok(true)
+    }
+    /// Whether any draw has been published (the first one initializes the buffer manager).
+    pub(crate) fn started(&self)->bool {self.draw>0}
+    pub(crate) fn first(&self)->bool {
+        self.profile==super::m3_profile::Profile::Legacy && self.draw==self.batch_count as u64
+    }
+    /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then
+    /// the VM must already be bound, no storage may grow and the slots must not be in flight
+    /// (the caller keeps the in-flight passes plus this batch within SLOTS).
+    pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,
+        commands:&[crate::m3::m3_submit::Command],base:usize,overlap:bool)->Result {
+        if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);}
+        if overlap {
+            if !self._binding.matches(vm) {return Err(EBUSY);}
+            for (index,&command) in commands.iter().enumerate() {
+                let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);};
+                let g=Self::geometry(r,self.clusters)?;
+                let slot=Self::at(base,index);
+                if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64
+                    || g.tilemap_bytes>self.passes[slot].get(P::Tilemap).size() as u64 {return Err(EBUSY);}
+            }
+        }
+        if commands.is_empty() || commands.len()>pass_layout::SLOTS {return Err(E2BIG);}
+        self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?;
+        // Validate every pass before changing any queue or completion state.
+        for &command in commands {
+            let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);};
+            Self::geometry(r,self.clusters)?;
+            crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?;
+        }
+        if !self._binding.matches(vm) {
+            let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) {
+                self.cached_views.remove(index).map_err(|_|EIO)?
+            } else {
+                let binding=uat.bind(vm)?;
+                let mut views=KVec::new();let mut aliases=KVec::new();
+                for i in 0..s::COUNT {
+                    let a=s::allocation_for(self.profile,i).map_err(|_|EINVAL)?;
+                    if a.gpu_alias.is_some() {aliases.push(self.objects[i].map_gpu_view(vm)?,GFP_KERNEL)?;}
+                    if a.space==Space::ClientGpu {
+                        views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?;
+                    }
+                }
+                for (slot,pass) in self.passes.iter_mut().enumerate() {
+                    for field in pass_layout::FIELDS {
+                        let a=pass_layout::allocation_for(self.profile,slot,field,self.clusters).map_err(|_|EINVAL)?;
+                        if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;}
+                        if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;}
+                    }
+                }
+                CachedViews{binding,aliases,views}
+            };
+            // Finish fallible allocations before replacing the current owner.
+            let old_views=KVec::with_capacity(next.views.len(),GFP_KERNEL)?;
+            self.cached_views.reserve(1,GFP_KERNEL)?;
+            if self.cached_views.len()==3 {drop(self.cached_views.remove(0).map_err(|_|EIO)?);}
+            let mut old=CachedViews {
+                binding:core::mem::replace(&mut self._binding,next.binding),
+                aliases:core::mem::replace(&mut self._aliases,next.aliases),
+                views:old_views,
+            };
+            for (i,view) in next.views {
+                old.views.push((i,self.view_owner(i).swap_gpu_view(view)?),GFP_KERNEL)?;
+            }
+            self.cached_views.push(old,GFP_KERNEL)?;
+        }
+        // Growth is legal only after exact retirement, and before append.
+        // Cached VM mappings must be dropped before freeing the old backing.
+        // Partial allocation failure leaves retired queue/stamp state intact;
+        // already-grown buffers are reusable on the next submission.
+        for (index,&command) in commands.iter().enumerate() {
+            let slot=Self::at(base,index);
+            let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);};
+            let g=Self::geometry(r,self.clusters)?;
+            for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] {
+                let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?;
+                if size>self.passes[slot].get(field).size() {
+                    let buffer=Buffer::tiler(dev,uat.kernel_vm(),vm,size)?;
+                    self.cached_views.clear();
+                    dev_info!(dev.as_ref(),"M3 render arena growth: slot={} object={} {} -> {} bytes GPU={:#x}\n",
+                        slot,base,self.passes[slot].get(field).size(),size,buffer.gpu_va()?);
+                    *self.passes[slot].get_mut(field)=buffer;
+                }
+            }
+        }
+        self.checkpoint=(self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base);
+        self.batch_count=0;
+        self.base=base;
+        Ok(())
+    }
+    fn publish_counts(&mut self, draw:u64) -> Result {
+        if self.profile==super::m3_profile::Profile::Extended {
+            for offset in [0,8] {self.objects[s::EVENT_COUNT].u64(offset,draw)?;}
+            for counter in [TA_POOL_COUNTER,FRAGMENT_POOL_COUNTER] {self.objects[counter].u64(0,draw)?;}
+        } else {
+            self.objects[s::BM_COUNTER].u32(0,draw as u32)?;
+            self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?;
+        }
+        Ok(())
+    }
+
+    /// Restore producer state only when no batch is in flight. Running firmware can
+    /// consume append's WRITE updates without a new doorbell, so overlap cannot abort.
+    pub(crate) fn abort_batch(&mut self)->Result {
+        (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint;
+        self.publish_counts(self.draw)?;
+        for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() {
+            self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?;
+        }
+        crate::m3::agx_memory::publish();
+        Ok(())
+    }
+    /// Append only into caller-reserved free pass slots. With overlap, WRITE updates
+    /// are immediately visible to running firmware, even before a new doorbell.
+    /// Each append selects disjoint host-mutated pass storage. A TA dependency
+    /// retains full fragment-to-next-TA ordering for arbitrary resource hazards.
+    pub(crate) fn append(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,coalesce_stamps:bool,timestamps:[bool;2])->Result {
+        self.append_after(r,usc,coalesce_stamps,None,timestamps)
+    }
+    pub(crate) fn completion_dependency(&self)->Result<sync::CompletionDependency> {
+        Ok(sync::CompletionDependency {
+            stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?,
+            value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1,
+        })
+    }
+    pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,
+        coalesce_stamps:bool,predecessor:Option<sync::CompletionDependency>,timestamps:[bool;2])->Result {
+        if predecessor.is_some_and(|p|p.event!=2) {return Err(EINVAL);}
+        if self.batch_count>=pass_layout::SLOTS {
+            pr_err!("M3 render batch limit: batch={} slots={}\n",
+                self.batch_count,pass_layout::SLOTS);
+            return Err(E2BIG);
+        }
+        self.slot=Self::at(self.base,self.batch_count);
+        self.stage_timestamps=timestamps;
+        // Retain the conservative stage-scope permission. Only a pass after the
+        // first of its batch may overlap its predecessor's stages. Explicit
+        // start timestamps keep the original full-order execution boundary.
+        let early=predecessor.is_none() && self.batch_count>0 && crate::m3::m3_params::early_tiling()
+            && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0
+            && r.fragment_timestamps.start.handle==0;
+        if early {self.early_count+=1;}
+        // Only reset per-draw objects. Pool allocators, PB lists, queues,
+        // notification state and previous stamps retain firmware ownership.
+        self.passes[self.slot].reset(&Self::geometry(r,self.clusters)?)?;
+        self.init_bm.fill(0)?; // retain the prior shared InitBM reset before encoding
+        self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?;
+        self.batch_count+=1;
+        let draw=self.draw;
+        self.publish_counts(draw)?;
+        if self.profile==super::m3_profile::Profile::Extended {
+            let pass=&self.passes[self.slot];
+            let scratch=pass.get(P::FragmentScratch);
+            let scratch_gpu=GpuRegion::new(scratch.gpu_va()?,scratch.size()).map_err(|_|EINVAL)?;
+            let scratch_fw=queue::FirmwareVa::new(scratch.va()).map_err(|_|EINVAL)?;
+            let list=pass.get(P::SceneUser);
+            let list=GpuRegion::new(list.gpu_va()?,list.size()).map_err(|_|EINVAL)?;
+            let metadata=pass.get(P::Scene);
+            let owner=Region::new(metadata.va(),metadata.size()).map_err(|_|EINVAL)?;
+            let mut bytes=[0;scene::EXTENDED_SIZE];
+            scene::encode_extended(&mut bytes,owner,scratch_gpu,scratch_fw,list).map_err(|_|EINVAL)?;
+            self.passes[self.slot].get_mut(P::Scene).write(0,&bytes)?;
+        } else {
+        let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4;
+        let scene_list = &mut self.objects[s::BM_SCENES];
+        if entry_offset + 4 > scene_list.size() { return Err(ERANGE); }
+        let manager_entry = queue::FirmwareVa::new(scene_list.va().checked_add(entry_offset as u64)
+            .ok_or(EOVERFLOW)?).map_err(|_| EINVAL)?;
+        let manager_misc = queue::FirmwareVa::new(self.objects[MANAGER_MISC].va()).map_err(|_| EINVAL)?;
+        let user_buffer = self.passes[self.slot].get_mut(P::SceneUser);
+        let user_buffer = scene::UserBuffer::new(user_buffer.gpu_va()?, user_buffer.size()).map_err(|_| EINVAL)?;
+        let mut bytes = [0; scene::SIZE];
+        scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?;
+        self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?;
+        }
+        self.prepare_tiler(r,usc,draw)?;
+        self.patch_context()?;
+        self.prepare_fragment(r,usc,draw)?;
+        self.prepare_sequences(draw)?;
+        self.prepare_sync(early, draw, predecessor)?;
+        // Encode before either ring WRITE exposes the command to firmware.
+        // The caller retains every pass owner until the final flushed stamps,
+        // queues, events and engine-idle checks retire this entire batch.
+        if coalesce_stamps {
+            self.passes[self.slot].get_mut(P::TilerCommand).u32(tiler::FLUSH_STAMPS,0)?;
+            self.passes[self.slot].get_mut(P::FragmentCommand).u32(fragment::FLUSH_STAMPS,0)?;
+        }
+        let dep=P::TilerDependency;
+        let fs_dep=P::FragmentDependency;
+        let ta=self.passes[self.slot].get_mut(P::TilerCommand).va();
+        let fragment=self.passes[self.slot].get_mut(P::FragmentCommand).va();
+        let barrier=self.passes[self.slot].get_mut(P::TilerToFragment).va();
+        let dependency=self.passes[self.slot].get(dep).va();
+        let init=self.init_bm.va();
+        let ta_commands=[init,dependency,ta];
+        let fs_commands=[self.passes[self.slot].get(fs_dep).va(),barrier,fragment];
+        for (stage,ring,pointers,addresses) in [
+            (0,s::TA_RING,s::TA_POINTERS,&ta_commands[usize::from(draw!=1)..]),
+            (1,s::FRAGMENT_RING,s::FRAGMENT_POINTERS,&fs_commands[usize::from(!early)..])] {
+            for &address in addresses {
+                self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?;
+                self.heads[stage]=(self.heads[stage]+1)%0x500;
+            }
+            crate::m3::agx_memory::publish();
+            self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?;
+        }
+        crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(())
+    }
+    /// Emitted only after both events, stamp/queue checks, clear faults,
+    /// idle engines and consumed firmware pipes have been checked by Runtime.
+    /// The existing TA/fragment descriptors request completion stamp flushes.
+    pub(crate) fn progress(&mut self,dev:&driver::AsahiDrmDevice)->Result {
+        // Keep summaries sparse at high throughput, but emit at least once a
+        // second while completed batches advance. Batched draw counts can skip
+        // multiples of 128; a count-only gate can starve the watchdog observer.
+        if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            return Ok(());
+        }
+        let now = <kernel::time::Monotonic as kernel::time::ClockSource>::ktime_get();
+        if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000
+            && self.draw != 1 && self.draw % 128 != 0
+            && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            return Ok(());
+        }
+        self.last_progress_ns = now;
+        dev_info!(dev.as_ref(),"M3_RETIRED draw={} ta={:x}/{:x} fragment={:x}/{:x} queues={}/{} read={}/{} idle=1 cache_flush=1 events=2 faults=0 ordered_barrier=1 batch={} early={}\n",
+            self.draw,self.objects[s::TA_STAMP].read_u32(0)?,self.objects[s::TA_FW_STAMP].read_u32(0)?,
+            self.objects[s::FRAGMENT_STAMP].read_u32(0)?,self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?,
+            self.objects[s::TA_POINTERS].read_u32(queue::DONE)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::DONE)?,
+            self.objects[s::TA_POINTERS].read_u32(queue::READ)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::READ)?,self.batch_count,self.early_count);
+        Ok(())
+    }
+    pub(crate) fn queues(&self)->[u64;2] {[self.objects[s::TA_QUEUE].va(),self.objects[s::FRAGMENT_QUEUE].va()]}
+    /// Read retained command bytes only after exact batch retirement.
+    pub(crate) fn stamp_flush_masks(&mut self,base:usize,count:usize)->Result<[u16;2]> {
+        if base>=pass_layout::SLOTS || count==0 || count>pass_layout::SLOTS {return Err(EINVAL);}
+        let mut masks=[0;2];
+        for slot in 0..count {
+            let pass=&mut self.passes[Self::at(base,slot)];
+            for (stage,field,offset) in [(0,P::TilerCommand,tiler::FLUSH_STAMPS),
+                (1,P::FragmentCommand,fragment::FLUSH_STAMPS)] {
+                if pass.get_mut(field).read_u32(offset)?!=0 {masks[stage]|=1u16<<slot;}
+            }
+        }
+        Ok(masks)
+    }
+    /// Read the fragment host and firmware stamps without changing retirement.
+    pub(crate) fn fragment_stamp_parts(&mut self)->Result<[bool;2]> {
+        let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw);
+        Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value,
+            self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value])
+    }
+    /// Observe each completion condition across both render queues independently.
+    pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> {
+        let mut done=[true;4];
+        for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]),
+            (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] {
+            done[0]&=self.objects[stamp].read_u32(0)?==value;
+            done[1]&=self.objects[fw].read_u32(0)?==value;
+            done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head);
+            done[3]&=self.objects[pointers].read_u32(queue::READ)?==u32::from(head);
+        }
+        Ok(done)
+    }
+    pub(crate) fn complete(&mut self)->Result<bool> {
+        for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]),
+            (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] {
+            if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value
+                || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head)
+                || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);}
+        }
+        if self.profile==super::m3_profile::Profile::Extended && self.draw!=0 {
+            for pool in UMA_POOLS {
+                if self.objects[pool.manager].read_u64(0x54)?!=self.draw {return Ok(false);}
+            }
+        }
+        Ok(true)
+    }
+    pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {
+        // Retain allocator state as well as payloads: an exhausted parameter
+        // pool can stall TA without producing an MMU fault.
+        for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] {
+            for offset in (0..bytes).step_by(4) {
+                let value=self.objects[index].read_u32(offset)?;
+                dev_info!(dev.as_ref(),"M3_PB object={} offset={:#x} value={:#x}\n",index,offset,value);
+            }
+        }
+        dev_info!(dev.as_ref(),"M3 fault render VM={} slot={} root={:#x} draw={} batch={}\n",self._binding.vm_id(),self._binding.slot(),self._binding.root(),self.draw,self.batch_count);
+        for index in 0..self.batch_count {
+            let slot=Self::at(self.base,index);
+            let command=self.passes[slot].get_mut(P::TilerCommand);
+            for index in 0..tiler::REGISTER_COUNT {
+                let offset=tiler::REGISTERS+index*tiler::REGISTER_STRIDE;
+                if command.read_u32(offset)?==0x1c880 {
+                    dev_info!(dev.as_ref(),"M3 fault pass slot={} compact_vdm={:#x}\n",slot,command.read_u64(offset+4)?);
+                }
+            }
+        }
+        if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) {
+        // Preserve historic diagnostic IDs while selecting named owners.
+        for (index,bytes) in s::fault_buffers() {
+            let b=match index {s::TILEMAP_LABEL=>self.passes[0].get_mut(P::Tilemap),
+                s::HEAP_METADATA_LABEL=>self.passes[0].get_mut(P::HeapMetadata),_=>&mut self.objects[index]};
+            for offset in (0..bytes).step_by(16) {
+                let lo=b.read_u64(offset)?;
+                let hi=b.read_u64(offset+8)?;
+                if lo!=0 || hi!=0 {dev_info!(dev.as_ref(),"M3_BUFFER object={} offset={:#x} {:016x} {:016x}\n",index,offset,lo,hi);}
+            }
+        }
+        }
+        for (name,stamp,fw,pointers) in [("TA",s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS),
+            ("fragment",s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS)] {
+            dev_info!(dev.as_ref(),"M3 {} stamps={:#x}/{:#x} done={} read={}\n",name,
+                self.objects[stamp].read_u32(0)?,self.objects[fw].read_u32(0)?,
+                self.objects[pointers].read_u32(queue::DONE)?,self.objects[pointers].read_u32(queue::READ)?);
+        } Ok(())
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs
index f9a0483e77b21..3d421e9906692 100644
--- a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs
@@ -184,3 +184,171 @@ impl Fragment {
         Ok(if timed {LENGTH} else {without_timestamps(bytes,TIMESTAMP_START,FINALIZE,LENGTH,0xa4)})
     }
 }
+
+impl Owners {
+    pub(crate) fn encode_extended(&self, fragment:bool,
+        auxiliary:super::m3_compute_layout::GpuVa, serial:u64, output:&mut[u8]) -> Result<usize,Error> {
+        if self.context>=64 || serial==0 || self.stamp==0 {return Err(Error::Address);}
+        let command=self.command.at(0,if fragment {0xca0}else{0x940})?;
+        let sequence=self.sequence.at(0,STORAGE)?;
+        let activity=self.stats.at(0,8)?;
+        let scene_tail=self.scene.get().checked_add(0x54).ok_or(Error::Address)?;
+        let (clock,pair,aux_time,elapsed)=if fragment {(0xc08,0xc10,0xc70,0xb84)}else{(0x8c0,0x8c8,0x928,0x83c)};
+        let first=if fragment {0x1e4}else{0x1cc};
+        let finalize=if fragment {0x280}else{0x268};
+        let pool=if fragment {0x1b0}else{0x190};
+        let length=if fragment {0x370}else{0x300};
+        let uuid=if fragment {fragment::UUID}else{ta::UUID};
+        let bytes=output.get_mut(..STORAGE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        word(bytes,0,if fragment {7}else{5});
+        pointer(bytes,0x14,command+if fragment {0x80}else{0x40});
+        pointer(bytes,pool,self.pool.get());
+        pointer(bytes,pool+0x18,auxiliary.get());
+        pointer(bytes,pool+0x28,serial);
+        word(bytes,pool+0x30,u32::from(fragment));
+        if fragment {
+            pointer(bytes,0x1c,self.scene.get());
+            pointer(bytes,0x24,activity);
+            pointer(bytes,0x2c,command+0xb70);
+            pointer(bytes,0x34,scene_tail);
+            pointer(bytes,0x3c,command+0xbb0);
+            pointer(bytes,0x44,command+0xbb4);
+            pointer(bytes,0x4c,self.queue.get());
+            pointer(bytes,0x54,command);
+            word(bytes,0x5c,self.context);
+            word(bytes,0x60,1);
+            pointer(bytes,0x7c,command+0xa58);
+            pointer(bytes,0x84,command+0xbf0);
+            word(bytes,0xa0,uuid);
+        } else {
+            pointer(bytes,0x1c,self.manager.get());
+            pointer(bytes,0x24,self.scene.get());
+            pointer(bytes,0x2c,activity);
+            pointer(bytes,0x34,self.queue.get());
+            pointer(bytes,0x3c,command+0x85c);
+            word(bytes,0x44,self.context);
+            word(bytes,0x48,1);
+            pointer(bytes,0x64,command+0x784);
+            pointer(bytes,0x6c,command+0x8a8);
+            word(bytes,0x7c,uuid);
+        }
+        for (at,opcode,selected) in [(first,0x8000_0003,pair),(first+0x50,3,pair+8)] {
+            word(bytes,at,opcode);
+            pointer(bytes,at+4,command+clock);
+            pointer(bytes,at+0xc,command+pair);
+            pointer(bytes,at+0x14,command+selected);
+            pointer(bytes,at+0x1c,self.queue.get());
+            pointer(bytes,at+0x2c,command+aux_time);
+            pointer(bytes,at+0x34,command+elapsed);
+            pointer(bytes,at+0x44,uuid.into());
+        }
+        word(bytes,first+0x4c,1);
+        word(bytes,finalize,if fragment {8}else{6});
+        if fragment {
+            word(bytes,finalize+4,uuid);
+            pointer(bytes,finalize+0xc,self.fw_stamp.get());
+            word(bytes,finalize+0x14,self.stamp);
+            pointer(bytes,finalize+0x1c,self.scene.get());
+            pointer(bytes,finalize+0x24,self.manager.get());
+            word(bytes,finalize+0x2c,1);
+            pointer(bytes,finalize+0x30,activity);
+            pointer(bytes,finalize+0x38,command+0xbb0);
+            pointer(bytes,finalize+0x40,command+0xbb4);
+            pointer(bytes,finalize+0x48,command+0xb70);
+            pointer(bytes,finalize+0x50,self.queue.get());
+            pointer(bytes,finalize+0x58,command);
+            word(bytes,finalize+0x60,self.context);
+            pointer(bytes,finalize+0x64,command+0xa58);
+            pointer(bytes,finalize+0x98,sequence+pool as u64);
+            word(bytes,finalize+0xa0,(-(finalize as i32))as u32);
+            pointer(bytes,finalize+0xa5,command+0xc60);
+            word(bytes,finalize+0xad,1);
+            bytes[finalize+0xb1]=1;
+            // J613 terminates this record after the 16-bit event field.
+            bytes[finalize+0xb2..finalize+0xb4].copy_from_slice(&1u16.to_le_bytes());
+            word(bytes,finalize+0xb4,0x4000_0002);
+        } else {
+            pointer(bytes,finalize+4,self.scene.get());
+            pointer(bytes,finalize+0xc,self.manager.get());
+            pointer(bytes,finalize+0x14,activity);
+            pointer(bytes,finalize+0x1c,self.queue.get());
+            pointer(bytes,finalize+0x24,command+0x85c);
+            word(bytes,finalize+0x2c,self.context);
+            pointer(bytes,finalize+0x34,command+0x784);
+            word(bytes,finalize+0x40,uuid);
+            pointer(bytes,finalize+0x48,self.fw_stamp.get());
+            word(bytes,finalize+0x50,self.stamp);
+            pointer(bytes,finalize+0x78,sequence+pool as u64);
+            word(bytes,finalize+0x80,(-(finalize as i32))as u32);
+            pointer(bytes,finalize+0x85,command+0x918);
+            bytes[finalize+0x8d]=1;
+            word(bytes,finalize+0x94,0x4000_0002);
+        }
+        Ok(length)
+    }
+}
+
+#[cfg(test)]
+mod native_tests {
+    use super::*;
+    #[test] fn j613_fragment_ends_before_the_m4_tail_and_keeps_relative_restart() {
+        let fw=|index:u64|FirmwareVa::new(0xffff_fc25_2000_0000+index*0x4000).unwrap();
+        let o=Owners {command:Region::new(fw(0).get(),0xca0).unwrap(),sequence:Region::new(fw(1).get(),STORAGE).unwrap(),
+            stats:Region::new(fw(2).get(),0x1000).unwrap(),notifier:Region::new(fw(3).get(),0x200).unwrap(),
+            queue:fw(4),scene:fw(5),manager:fw(6),pool:fw(7),fw_stamp:fw(8),context:3,stamp:0x3d000100};
+        let mut output=[0xa5;STORAGE+8];
+        let length=o.encode_extended(true,super::super::m3_compute_layout::GpuVa::new(0x72_3000_0200).unwrap(),1,&mut output).unwrap();
+        assert_eq!(length,0x370);
+        let word=|at|u32::from_le_bytes(output[at..at+4].try_into().unwrap());
+        assert_eq!(word(0x1e4),0x8000_0003);
+        assert_eq!(word(0x230),1);
+        assert_eq!(word(0x234),3);
+        assert_eq!(word(0x280),8);
+        assert_eq!(word(0x320),(-0x280i32)as u32);
+        assert_eq!(word(0x334),0x4000_0002);
+        assert_eq!(&output[STORAGE..],&[0xa5;8]);
+    }
+}
+
+/// Native start records reserve one packed attachment list per engine. The
+/// matching finalizer flag enables completion visibility for that same list.
+pub(crate) const fn extended_attachment_offsets(fragment:bool)->(usize,usize) {
+    if fragment {(0xa8,0x324)} else {(0x88,0x2ec)}
+}
+
+/// Send both pieces through the same bounded writer used for firmware storage.
+#[cfg(any(test, CONFIG_DRM_ASAHI_M3_T8122))]
+pub(crate) fn patch_extended_attachments<E>(fragment:bool,
+    attachments:&super::agx_attachments::Attachments,
+    mut write:impl FnMut(usize,&[u8])->Result<(),E>)->Result<(),E> {
+    let (start,flag)=extended_attachment_offsets(fragment);
+    write(start,&attachments.0)?;
+    write(flag,&[u8::from(attachments.count()!=0)])
+}
+
+#[cfg(test)]
+mod attachment_tests {
+    use super::*;
+    use super::super::agx_attachments::{Attachments,SIZE};
+    #[test] fn each_engine_keeps_its_list_and_can_clear_a_previous_list() {
+        for (fragment,start,flag,pool) in [(false,0x88,0x2ec,0x190),(true,0xa8,0x324,0x1b0)] {
+            let mut bytes=[0xa5;STORAGE];
+            let list=Attachments::new(&[(0x100_0040,0x180),(0x200_0000,0x4000)]).unwrap();
+            let mut apply=|list:&Attachments|patch_extended_attachments(fragment,list,|at,data|{
+                bytes.get_mut(at..at+data.len()).ok_or(Error::Size)?.copy_from_slice(data);Ok::<(),Error>(())
+            });
+            apply(&list).unwrap();
+            assert_eq!(&bytes[start..start+SIZE],&list.0);
+            assert_eq!(bytes[flag],1);
+            assert_eq!(&bytes[pool..pool+8],&[0xa5;8]);
+            patch_extended_attachments(fragment,&Attachments::EMPTY,|at,data|{
+                bytes[at..at+data.len()].copy_from_slice(data);Ok::<(),Error>(())
+            }).unwrap();
+            assert!(bytes[start..start+SIZE].iter().all(|b|*b==0));
+            assert_eq!(bytes[flag],0);
+            assert_eq!(bytes[start-1],0xa5);
+            assert_eq!(bytes[start+SIZE],0xa5);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs
new file mode 100644
index 0000000000000..7d34305250018
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! M3 GPU resources admitted against the measured boot contract (J514S on T6030).
+pub(crate) use crate::m3::agx_resources::{Region, Resources};
+
+use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*};
+
+/// Live overlays are not inserted in OF's boot-time reserved_mem registry.
+/// Keep the ordinary lookup for every static handoff. A no-map node is
+/// accepted only when its range is outside all Linux System RAM.
+pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result<Resource> {
+    if let Ok(r) = node.reserved_mem_region_to_resource_byname(name) {
+        return Ok(r);
+    }
+    if let Some(found) = crate::m3::m3_board::static_region(node, name) {
+        return found.map(|(r, _)| r);
+    }
+    // Only for a GPU node whose compatible is exactly that of an M3 SoC table.
+    let compatible: KVec<u8> = node.get_property(c_str!("compatible"))?;
+    let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?;
+    if !crate::m3::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) {
+        return Err(EINVAL);
+    }
+    let names: KVec<u8> = node.get_property(c_str!("memory-region-names"))?;
+    let index = names.split(|b| *b == 0).position(|n| n == name.to_bytes()).ok_or(EINVAL)?;
+    let region = node.parse_phandle(c_str!("memory-region"), index).ok_or(EINVAL)?;
+    let nomap: KVec<u8> = region.get_property(c_str!("no-map"))?;
+    let reg: KVec<u64> = region.get_property(c_str!("reg"))?;
+    if !nomap.is_empty() || reg.len()!=2 { return Err(EINVAL); }
+    let (base,size)=(reg[0],reg[1]);
+    if base < 1<<40 || size == 0 || (base|size)&0x3fff != 0
+        || base.checked_add(size).ok_or(EOVERFLOW)? > 1<<42 { return Err(EINVAL); }
+    // SAFETY: read-only resource-tree query with checked numeric bounds.
+    if unsafe { bindings::region_intersects(base, size.try_into()?, bindings::IORESOURCE_SYSTEM_RAM as _, 0) } != bindings::REGION_DISJOINT as i32 {
+        return Err(EBUSY);
+    }
+    let raw = bindings::resource {
+        start: base, end: base+size-1, flags: bindings::IORESOURCE_MEM as _,
+        ..Default::default()
+    };
+    // SAFETY: Resource is repr(transparent) over Opaque<resource>, which
+    // preserves layout. The initialized descriptor is owned, with no pointers
+    // into temporary data; its range is checked above.
+    Ok(unsafe { core::mem::transmute::<bindings::resource, Resource>(raw) })
+}
+
+pub(crate) fn from_device(
+    pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+    soc: &crate::m3::m3_soc::Soc,
+) -> kernel::error::Result<Resources> {
+    crate::m3::m3_board::admit(pdev, soc)
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs
new file mode 100644
index 0000000000000..ba8ad365ce20a
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! RTKit callbacks for the identified J514S firmware. The firmware advertises
+//! its crash storage by physical address inside the reserved data segment.
+
+use crate::m3::agx_resources::Region;
+use core::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use kernel::{
+    c_str, impl_has_hr_timer,
+    device::Core,
+    sync::Completion,
+    io::{mem::{Mem, MemFlag}, IoSysMap, Region as IoRegion, SysMem},
+    platform,
+    prelude::*,
+    soc::apple::rtkit,
+    sync::{aref::ARef, Arc, ArcBorrow, SetOnce},
+    time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback,
+        HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}},
+};
+
+/// IRQ-safe notification tokens for the single GPU completion worker.
+#[pin_data]
+pub(crate) struct EventWait {
+    count: AtomicU64,
+    wait_us: AtomicU64,
+    overlap_cpu: AtomicBool,
+    cpu_preparations: AtomicU64,
+    sleeps: AtomicU64,
+    timer_wakes: AtomicU64,
+    #[pin]
+    arrived: Completion,
+    #[pin]
+    timer: HrTimer<Self>,
+}
+
+impl EventWait {
+    fn new() -> Result<Arc<Self>> {
+        Arc::pin_init(
+            pin_init!(EventWait {
+                count: AtomicU64::new(0),
+                wait_us: AtomicU64::new(1000),
+                overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()),
+                cpu_preparations: AtomicU64::new(0),
+                sleeps: AtomicU64::new(0),
+                timer_wakes: AtomicU64::new(0),
+                arrived <- Completion::new(),
+                timer <- HrTimer::new(),
+            }),
+            GFP_KERNEL,
+        )
+    }
+
+    fn record(&self) {
+        self.count.fetch_add(1, Ordering::Release);
+        self.arrived.complete();
+    }
+
+    /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never
+    /// stamps, pipe retirement, event counts, fences or the two-second timeout.
+    /// Atomic publication permits root to compare policies in the same boot.
+    pub(crate) fn set_wait_us(&self, us: u64) -> Result {
+        if !(25..=1000).contains(&us) { return Err(EINVAL); }
+        self.wait_us.store(us, Ordering::Relaxed);
+        Ok(())
+    }
+
+    pub(crate) fn wait_ceiling_us(&self) -> u64 {
+        self.wait_us.load(Ordering::Relaxed)
+    }
+
+    pub(crate) fn cpu_overlap(&self) -> bool {
+        self.overlap_cpu.load(Ordering::Relaxed)
+    }
+    pub(crate) fn set_cpu_overlap(&self, enabled: bool) {
+        self.overlap_cpu.store(enabled, Ordering::Relaxed);
+    }
+
+    pub(crate) fn record_cpu_preparation(&self) {
+        self.cpu_preparations.fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn snapshot(&self) -> (u64, u64, u64, u64, u64) {
+        (self.wait_us.load(Ordering::Relaxed), self.count.load(Ordering::Acquire),
+         self.sleeps.load(Ordering::Relaxed), self.timer_wakes.load(Ordering::Relaxed),
+         self.cpu_preparations.load(Ordering::Relaxed))
+    }
+
+    /// Sleep until a notification later than `seen` has been recorded or
+    /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250,
+    /// even when the GPU stamp arrives just after its notification. The timer
+    /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting.
+    /// Only the single completion worker may call this method.
+    pub(crate) fn wait_past(self: &Arc<Self>, seen: u64, wait_us: u64) -> bool {
+        // Events can arrive while the worker is polling. Discard stale wakeup
+        // tokens before checking the sequence. A concurrent event either moves
+        // the sequence or leaves a token, including between the check and wait.
+        // Bound draining so a busy interrupt source cannot hold this worker.
+        for _ in 0..64 {
+            if !self.arrived.try_wait_for_completion() {
+                if self.count.load(Ordering::Acquire) != seen { return true; }
+                // The handle keeps EventWait alive and cancels synchronously
+                // on every return, before a subsequent wait can arm the timer.
+                self.sleeps.fetch_add(1, Ordering::Relaxed);
+                let timer = self.clone().start(Delta::from_micros(
+                    wait_us.clamp(25, 1000) as i64));
+                // Retain one tick as a backup bound; normally either the real
+                // interrupt or our high-resolution timer supplies the token.
+                self.arrived.wait_for_completion_timeout(1);
+                drop(timer);
+                return self.count.load(Ordering::Acquire) != seen;
+            }
+        }
+        // Resnapshot GPU progress before draining another batch of tokens.
+        true
+    }
+}
+
+impl HrTimerCallback for EventWait {
+    type Pointer<'a> = Arc<Self>;
+
+    fn run(this: ArcBorrow<'_, Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart {
+        // Wake only: a timer expiry is not a firmware completion notification.
+        this.timer_wakes.fetch_add(1, Ordering::Relaxed);
+        this.arrived.complete();
+        HrTimerRestart::NoRestart
+    }
+}
+
+impl_has_hr_timer! {
+    impl HasHrTimer<Self> for EventWait {
+        mode: RelativeMode<Monotonic>, field: self.timer
+    }
+}
+
+pub(crate) struct State {
+    dev: ARef<platform::Device>,
+    drm: crate::driver::AsahiDevRef,
+    data: Region,
+    pub(crate) health: Arc<Health>,
+    claimed: AtomicBool,
+    /// Firmware event notifications received on the application endpoint.
+    /// The firmware sends one after a command's completion processing.
+    pub(crate) event_messages: AtomicU64,
+    pub(crate) events: Arc<EventWait>,
+    /// The completion worker, published once the runtime it services exists. The DRM device
+    /// carries no driver data for it on 7.2, so the runtime's own state owns it.
+    pub(crate) completion: SetOnce<Arc<crate::m3::m3_completion::Completion>>,
+}
+
+/// Shared status contains no device references, so DRM files may retain it
+/// after runtime teardown without retaining MMIO or creating a DRM cycle.
+pub(crate) struct Health {
+    crashed: AtomicBool,
+    mapped: AtomicBool,
+    failed: AtomicBool,
+    progress: crate::m3::agx_host_progress::Progress,
+    pub(crate) timing: crate::m3::agx_timing_stats::Stats,
+}
+
+impl Health {
+    pub(crate) fn healthy(&self) -> bool {
+        self.mapped.load(Ordering::Acquire)
+            && !self.crashed.load(Ordering::Acquire)
+            && !self.failed()
+    }
+
+    pub(crate) fn set_gpu_pending(&self, pending: bool) { self.progress.set_pending(pending); }
+
+    pub(crate) fn activity_snapshot(&self) -> (u64, u64, bool) {
+        let (generation, epoch) = self.progress.activity_snapshot();
+        (generation, epoch, self.healthy())
+    }
+
+    pub(crate) fn record_completion(&self) { self.progress.record_completion(); }
+
+    pub(crate) fn progress_snapshot(&self) -> (u64, u64, u64, bool) {
+        let (generation, count, timestamp) = self.progress.snapshot();
+        (generation, count, timestamp, self.healthy())
+    }
+
+    pub(crate) fn failed(&self) -> bool { self.failed.load(Ordering::Acquire) }
+
+    /// Whether RTKit reported a firmware crash.
+    pub(crate) fn crashed(&self) -> bool { self.crashed.load(Ordering::Acquire) }
+
+    pub(crate) fn mark_failed(&self) { self.failed.store(true, Ordering::Release); }
+}
+
+impl State {
+    pub(crate) fn new(dev: &platform::Device<Core<'_>>, drm: crate::driver::AsahiDevRef, data: Region) -> Result<Arc<Self>> {
+        Ok(Arc::new(
+            Self {
+                dev: dev.into(),
+                drm,
+                data,
+                health: Arc::new(Health {
+                    crashed: AtomicBool::new(false),
+                    mapped: AtomicBool::new(false),
+                    failed: AtomicBool::new(false),
+                    progress: crate::m3::agx_host_progress::Progress::new(),
+                    timing: crate::m3::agx_timing_stats::Stats::new(),
+                }, GFP_KERNEL)?,
+                claimed: AtomicBool::new(false),
+                event_messages: AtomicU64::new(0),
+                events: EventWait::new()?,
+                completion: SetOnce::new(),
+            },
+            GFP_KERNEL,
+        )?)
+    }
+
+    pub(crate) fn healthy(&self) -> bool {
+        self.health.healthy()
+    }
+}
+
+pub(crate) struct FirmwareBuffer {
+    state: Arc<State>,
+    mapping: Mem,
+    physical: usize,
+    offset: usize,
+    size: usize,
+}
+
+impl Drop for FirmwareBuffer {
+    fn drop(&mut self) {
+        // Includes allocation failure in RTKit's wrapper after shmem_map
+        // returned: readiness must not survive loss of the actual mapping.
+        self.state.health.mapped.store(false, Ordering::Release);
+    }
+}
+
+impl rtkit::Buffer for FirmwareBuffer {
+    fn iova(&self) -> Result<usize> {
+        Ok(self.physical)
+    }
+    fn buf(&mut self) -> Result<IoSysMap<'_, IoRegion>> {
+        // The view of 7.1.12's `Mem::iosys_map`: the bounded span inside the retained WB
+        // mapping, as normal memory.
+        if self.offset.checked_add(self.size).ok_or(EOVERFLOW)? > self.mapping.size() {
+            return Err(EINVAL);
+        }
+        // SAFETY: The span lies entirely inside the retained mapping (checked above).
+        let base = unsafe { self.mapping.ptr().add(self.offset) };
+        let region = IoRegion::ptr_try_from_raw_parts_mut(base, self.size)?;
+        // SAFETY: The span is mapped kernel memory for as long as `self.mapping`, which the
+        // returned view borrows through `&mut self`.
+        Ok(IoSysMap::Sys(unsafe { SysMem::new(region) }))
+    }
+}
+
+pub(crate) struct Operations;
+
+#[kernel::macros::vtable]
+impl rtkit::Operations for Operations {
+    type Data = Arc<State>;
+    type Buffer = FirmwareBuffer;
+
+    fn crashed(state: ArcBorrow<'_, State>, crashlog: Option<&[u8]>) {
+        state.health.crashed.store(true, Ordering::Release);
+        // Wake the serialized worker even if the firmware can no longer send
+        // its usual completion notification. Never take the runtime lock here.
+        state.events.record();
+        crate::m3::m3_completion::queue(&state);
+        dev_err!(
+            state.dev.as_ref(),
+            "M3 G15S: firmware crashed, retained crashlog bytes={}\n",
+            crashlog.map_or(0, |b| b.len())
+        );
+    }
+
+    fn shmem_map(
+        state: ArcBorrow<'_, State>,
+        physical: usize,
+        size: usize,
+    ) -> Result<FirmwareBuffer> {
+        // The only system buffer advertised by this admitted firmware is its
+        // preallocated crash log. Never reinterpret an arbitrary DVA as a PA.
+        let offset = physical
+            .checked_sub(state.data.base as usize)
+            .ok_or(EINVAL)?;
+        if size == 0
+            || (physical | size) & 0xfff != 0
+            || offset.checked_add(size).ok_or(EOVERFLOW)? > state.data.size as usize
+        {
+            return Err(EINVAL);
+        }
+        if state.claimed.swap(true, Ordering::AcqRel) {
+            return Err(EBUSY);
+        }
+        let node = state.dev.as_ref().of_node().ok_or(ENODEV)?;
+        let resource = crate::m3::m3_resources::reserved_resource(&node,c_str!("fw-data"))?;
+        if resource.start() != state.data.base || resource.size() != state.data.size {
+            return Err(EINVAL);
+        }
+        // SAFETY: The validated no-map data reservation belongs to this
+        // firmware session. This CPU mapping is retained by RTKit until ASC
+        // is stopped and callbacks drained; physical pages are never freed.
+        // An uncached CPU alias lets RTKit snapshot newly written crash data.
+        let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?;
+        dev_info!(
+            state.dev.as_ref(),
+            "M3 G15S: mapped firmware crash storage PA={:#x} size={:#x}\n",
+            physical,
+            size
+        );
+        state.health.mapped.store(true, Ordering::Release);
+        Ok(FirmwareBuffer {
+            state: state.into(),
+            mapping,
+            physical,
+            offset,
+            size,
+        })
+    }
+
+    fn recv_message(state: ArcBorrow<'_, State>, endpoint: u8, message: u64) {
+        // Firmware retries its event notification until the shared rings are
+        // drained, including after the final userspace job. Never take the
+        // runtime mutex in the RTKit callback: enqueue the serialized worker.
+        if endpoint == 0x20 && message == 0x0042_0000_0000_0000 {
+            // Record the IRQ-safe token before publishing the external count.
+            // A concurrent waiter sees either the moved sequence or a token.
+            state.events.record();
+            state.event_messages.fetch_add(1, Ordering::Release);
+            if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+                dev_info!(state.dev.as_ref(), "M3 G15S_TIMING event monotonic_ns={} messages={}\n",
+                    <kernel::time::Monotonic as kernel::time::ClockSource>::ktime_get(),
+                    state.event_messages.load(Ordering::Acquire));
+            }
+            crate::m3::m3_completion::queue(&state);
+            return;
+        }
+        dev_warn!(
+            state.dev.as_ref(),
+            "M3 G15S: RTKit application ep={:#x} message={:#018x}\n",
+            endpoint,
+            message
+        );
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs
new file mode 100644
index 0000000000000..e34236fd57a3f
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs
@@ -0,0 +1,1396 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+use core::{mem::ManuallyDrop, sync::atomic::Ordering};
+use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc,
+    time::{Instant, Monotonic, Delta, delay::fsleep}};
+use crate::{driver, mmu, m3::m3_device::Device, m3::m3_config::Config, m3::m3_rtkit};
+
+#[path = "m3_batch_policy.rs"]
+pub(crate) mod policy;
+const _: () = assert!(policy::SLOTS == crate::m3::m3_pass_layout::SLOTS);
+
+enum NativeJob {Compute(crate::m3::m3_compute::Compute),Render(crate::m3::m3_render::Render)}
+/// Completion-worker outcome.
+pub(crate) enum Service {Idle,Progress,Waiting(u64)}
+/// The published batch: its commands in order, as (packet, command index).
+struct Batch {
+    entries: KVec<(Arc<crate::m3::m3_submit::Packet>,usize)>,
+    index: usize, kind: usize, first: crate::m3::m3_submit::Command,
+    preparation_ns: i64, previous_events: u64, expected_events: u64,
+    start: Instant<Monotonic>, measure: bool, doorbell_counter_ns: i64,
+    /// Render: first pass slot and the batch's last draw number.
+    base: usize, last_draw: u64, publication_tick:u64,
+    polling_ns: i64, polls: u64, stamp_ns: i64, first_event_ns: i64, events_ns: i64,
+    completion_ns: [i64;4],
+    /// First successful stamp observations and the preceding incomplete reads, in 24 MHz ticks.
+    fragment_host_tick:u64, fragment_fw_tick:u64,
+    fragment_host_pending_tick:u64, fragment_fw_pending_tick:u64,
+    pipes_idle_tick:u64,
+    coalesced_stamps: u64,
+    mixed: Option<MixedBatch>,
+}
+struct MixedBatch {compute_index:usize, renders:usize, computes:usize, dependencies:usize}
+#[derive(Clone, Copy, Default)]
+struct GeometryTiming {
+    width: u16, height: u16, samples: u8, count: u64,
+    prepare_ns: i64, active_ns: i64, gpu_ns: u64,
+    tiling_ns: u64, fragment_ns: u64, max_fragment_ns: u64,
+    polling_ns: i64, retirement_ns: i64, sleeping_ns: i64, polls: u64,
+    stamp_ns:i64, first_event_ns:i64, events_ns:i64, pickup_ns:i64, tail_ns:i64,
+    completion_ns: [i64;4],
+}
+impl NativeJob {
+    fn complete(&mut self)->Result<bool> {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}}
+    fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}}
+    fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}}
+}
+struct Inner {
+    transport: rtkit::RtKit<m3_rtkit::Operations>,
+    state: Arc<m3_rtkit::State>,
+    jobs: KVec<NativeJob>,
+    packets: KVec<Arc<crate::m3::m3_submit::Packet>>,
+    /// Packets not yet (fully) published: (packet, next command index), most urgent first.
+    pending: KVec<(Arc<crate::m3::m3_submit::Packet>,usize)>,
+    /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1.
+    active: KVec<Batch>,
+    gpu_pending: bool,
+    active_power_last: Option<Instant<Monotonic>>,
+    active_power_samples: u32,
+    fault_captured: bool,
+    timing: [[i64;7];2],
+    render_batches: [u64;crate::m3::m3_pass_layout::SLOTS],
+    geometry: [GeometryTiming; 32],
+    geometry_overflow: u64,
+    /// Whether a T8122 job verdict line was already emitted for the current job, so the
+    /// execute-path fallback does not add a second one. T8122 only.
+    t8122_verdict: bool,
+    /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only.
+    boot_step: crate::m3::t8122_start::BootStep,
+    /// Packets that joined a batch started by an earlier packet (debug summary).
+    coalesced_packets: u64,
+    coalesced_render_stamps: u64,
+    mixed_batches:u64,
+    /// Recent measured render GPU time per pass by VM. Smoothed over batches;
+    /// replacing the least recently measured VM does not retain its mappings.
+    pass_cost: policy::PassCosts,
+    /// Latched only after the isolated InitBM draw fully retired.
+    render_initialized: bool,
+    config: Config,
+    uat: mmu::Uat,
+    /// The firmware's view of the host window (the shared L2 the firmware walks), kept in step
+    /// with the kernel VM's shadow window before the firmware is given new kernel addresses.
+    window: crate::g15::uat::Window,
+    drm: driver::AsahiDevRef,
+    device: Device,
+}
+impl Inner {
+    /// Latch before allocating diagnostics, once per runtime. Called under
+    /// its mutex even for errors outside the active-job polling loop.
+    fn capture_fault(&mut self, primary: Error) {
+        self.state.health.mark_failed();
+        if self.fault_captured { return; }
+        self.fault_captured = true;
+        // 7.1.12 also published a devcoredump of the firmware objects here; 7.2's kernel crate
+        // has no devcoredump binding, so the first error is only logged.
+        dev_err!(self.drm.as_ref(), "M3: first GPU failure: {:?} (GPU work pending: {})\n",
+            primary, self.gpu_pending);
+    }
+
+    /// Record a batch that failed or never retired. Mark the GPU failed first,
+    /// so that a failing diagnostic read cannot leave a job that never retired
+    /// behind a device that still reports itself healthy.
+    fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) {
+        self.capture_fault(primary);
+        if crate::m3::t8122_start::is_t8122(self.device.soc()) {
+            let pstate=self.device.pstate_register().ok();
+            // The job's first GPU start timestamp: nonzero means the firmware dispatched it,
+            // whatever the power state reads at the timeout.
+            let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]);
+            crate::m3::t8122_start::job_failed_verdict(self.drm.as_ref().as_ref(),primary,start_stamp,pstate,self.state.health.crashed());
+            self.t8122_verdict=true;
+        }
+        let events=self.state.event_messages.load(Ordering::Acquire);
+        if let Err(e) = self.jobs[index].log(&self.drm) {
+            dev_err!(self.drm.as_ref(), "M3 job diagnostics failed: {:?}\n", e);
+        }
+        if let Err(e) = self.device.log_engine_state(vm) {
+            dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e);
+        }
+        let _=self.config.log_recovery_state(&self.drm,events);
+        if crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault) {
+            if let Err(e)=self.resume_experiment(index, vm) {
+                dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e);
+            }
+        }
+    }
+    /// asahi.g15_debug bit 56. Wait up to 100 ms for the firmware to report
+    /// that it halted, resume it, and log every change of the halt words,
+    /// the engine state, the failed job's retirement, whether every queue
+    /// message was consumed and the event count for about half a second,
+    /// then the event ring entries the host has not consumed. The GPU stays
+    /// marked failed; the event ring's read index is not advanced.
+    fn resume_experiment(&mut self, index: usize, vm: &mmu::Vm) -> Result {
+        let start=Instant::<Monotonic>::now();
+        let mut halt=self.config.halt_state()?;
+        while halt.1==0 && start.elapsed()<Delta::from_millis(100) {
+            fsleep(Delta::from_millis(1));
+            halt=self.config.halt_state()?;
+        }
+        if halt.1==0 {
+            dev_info!(self.drm.as_ref(),"M3: resume-after-fault: the firmware did not halt within 100 ms (halt_count={} resume={}); not resuming\n",halt.0,halt.2);
+            return Ok(());
+        }
+        dev_info!(self.drm.as_ref(),"M3: resume-after-fault: the firmware halted (halt_count={} halted={}); clearing halted and setting resume\n",halt.0,halt.1);
+        self.config.resume_halted()?;
+        let resumed=Instant::<Monotonic>::now();
+        let mut last=None;
+        for _ in 0..500 {
+            let halt=self.config.halt_state()?;
+            // 0: idle and no fault bits, 1: an engine is busy, 2: fault bits set.
+            let engines=match self.device.check_idle() {Ok(())=>0u8,Err(e) if e==EBUSY=>1,Err(_)=>2};
+            let complete=self.jobs[index].complete().unwrap_or(false);
+            // 0: queues not drained, 1: every submitted queue message consumed, 2: read failed.
+            let pipes=match self.config.pipes_idle() {Ok(false)=>0u8,Ok(true)=>1,Err(_)=>2};
+            let events=self.state.event_messages.load(Ordering::Acquire);
+            let crashed=self.state.health.crashed();
+            let now=(halt,engines,complete,pipes,events,crashed);
+            if last!=Some(now) {
+                dev_info!(self.drm.as_ref(),"M3: resume-after-fault: +{}us: halt_count={} halted={} resume={} engines={} job_complete={} pipes_idle={} event_messages={} crashed={}\n",
+                    resumed.elapsed().as_nanos()/1000,halt.0,halt.1,halt.2,engines,complete,pipes,events,crashed);
+                last=Some(now);
+            }
+            if crashed {break;}
+            fsleep(Delta::from_millis(1));
+        }
+        self.jobs[index].log(&self.drm)?;
+        self.device.log_engine_state(vm)?;
+        self.config.log_event_ring(&self.drm)?;
+        let events=self.state.event_messages.load(Ordering::Acquire);
+        self.config.log_recovery_state(&self.drm,events)
+    }
+}
+impl Inner {
+    /// Fail every packet not yet fully published. Their fences signal with the error. One that no
+    /// batch ever included releases its VM job guard first, so its VM isn't quarantined for work
+    /// the firmware never saw. One that may have been published keeps it: a partly published
+    /// packet, or one whose batch `fail_active` has already drained from `active`.
+    fn fail_pending(&mut self, error: Error) {
+        for (packet,_) in self.pending.drain_all() {
+            if policy::release_unpublished(packet.may_be_published()) {packet.release_guard();}
+            packet.finish(Err(error));
+        }
+    }
+    /// A published batch failed or never retired: latch the fault, log it, and fail its
+    /// packets. The packets stay retained (with their VM job guards) until teardown, because
+    /// the firmware has not provably stopped using their memory.
+    fn fail_active(&mut self, error: Error) {
+        let _=self.config.performance_request(self.device.soc().chip_id,false);
+        if let Some((index,vm))=self.active.first().map(|b|(b.index,b.entries[0].0.vm.clone())) {
+            self.log_owners(error);
+            self.fail(index,&vm,error);
+        } else {self.capture_fault(error);}
+        for batch in self.active.drain_all() {Self::finish_all(&batch.entries,Err(error));}
+        self.fail_pending(error);
+    }
+    /// Name the processes whose published commands were in flight when the GPU failed. Batches
+    /// in flight share one VM, so this is normally one process; a VM shared between processes
+    /// names each submitter once (the first four).
+    fn log_owners(&self, error: Error) {
+        let commands:usize=self.active.iter().map(|b|b.entries.len()).sum();
+        let mut logged=[0i32;4];
+        let mut count=0;
+        for (packet,_) in self.active.iter().flat_map(|b|b.entries.iter()) {
+            let owner=&packet.owner;
+            if logged[..count].contains(&owner.pid) {continue;}
+            if count==logged.len() {break;}
+            dev_err!(self.drm.as_ref(),"M3 failed work owner: VM={} process {} pid {} thread {} (error {:?}, {} commands in flight)\n",
+                packet.vm.id(),owner.process(),owner.pid,owner.thread(),error,commands);
+            logged[count]=owner.pid;count+=1;
+        }
+    }
+    /// Hardware completion is separate from fence signaling: cancellation can win the
+    /// signal race while the DMA guard must survive until its final published batch retires.
+    fn packet_active(&self, packet:&Arc<crate::m3::m3_submit::Packet>)->bool {
+        self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet)))
+    }
+    fn retire_packets(&mut self,entries:&[(Arc<crate::m3::m3_submit::Packet>,usize)]) {
+        for (packet,_) in entries {
+            let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet));
+            if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();}
+        }
+        // Keep exactly active DMA owners. Successful retirement may precede a VM switch.
+        self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q))));
+    }
+    fn fail_front(&mut self,error:Error) {
+        if let Ok((packet,_))=self.pending.remove(0) {
+            if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();}
+            packet.finish(Err(error));
+        }
+    }
+    /// Signal every packet of `entries` (packets that continue past the batch included).
+    fn finish_all(entries:&[(Arc<crate::m3::m3_submit::Packet>,usize)],result:Result) {
+        for (packet,_) in entries {packet.finish(result);}
+    }
+    /// Choose the next batch from the front of `pending`: the front packet's next commands of
+    /// one engine, and for render also the leading render commands of following packets of the
+    /// same VM, up to the ordered-batch limit.
+    fn next_batch(&self,max:usize)->Result<KVec<(Arc<crate::m3::m3_submit::Packet>,usize)>> {
+        let mut entries=KVec::new();
+        let Some((packet,next))=self.pending.first() else {return Ok(entries);};
+        let mixed=policy::mixed_prefix(crate::m3::m3_params::mixed_batch(),
+            self.device.soc().chip_id,self.render_initialized,self.active.is_empty(),
+            packet.commands[*next..].iter().map(|c|matches!(c,crate::m3::m3_submit::Command::Render{..})));
+        if mixed>0 {
+            for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;}
+            return Ok(entries);
+        }
+        let render=matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..});
+        let limit=if render {
+            // The first render draw initializes the shared buffer manager (InitBM) and opens
+            // both queues. Publish it alone: appending later draws first advances the shared
+            // manager and event counters it starts from, and on J613 the second pass's
+            // fragment then fails with firmware error event 4 (draw=2 batch=2).
+            let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started()));
+            if started {crate::m3::m3_params::render_batch_size().min(max).max(1)} else {1}
+        } else {
+            crate::m3::m3_params::compute_batch_size()
+                .clamp(1,crate::m3::m3_compute_storage::SLOTS)
+        };
+        let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..});
+        let per_pass=self.pass_cost.get(packet.vm.id());
+        let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc());
+        let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum();
+        for (position,(candidate,start)) in self.pending.iter().enumerate() {
+            // Later packets join only render batches, only from their first command, and only
+            // for the VM the batch is bound to, while the batch's expected GPU time is within
+            // the budget.
+            if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;}
+            let mut index=*start;
+            while index<candidate.commands.len() && entries.len()<limit && same_engine(&candidate.commands[index]) {
+                if render && !policy::within_budget(per_pass,budget,in_flight+entries.len()) {return Ok(entries);}
+                entries.push((candidate.clone(),index),GFP_KERNEL)?;
+                index+=1;
+            }
+            // Stop at the batch limit or where a packet continues with other commands.
+            if entries.len()>=limit || index<candidate.commands.len() {break;}
+        }
+        Ok(entries)
+    }
+    /// Whether the front pending work may be published behind the in-flight batches, and if so
+    /// in which pass slot it starts and how many passes it may use.
+    fn pipeline_slot(&mut self)->Result<Option<(usize,usize)>> {
+        let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3::m3_pass_layout::SLOTS)));};
+        if !self.render_initialized {return Ok(None);}
+        let Some((packet,next))=self.pending.first() else {return Ok(None);};
+        if !matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..})
+            || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);}
+        let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum();
+        let Some(room)=policy::pipeline_room(crate::m3::m3_params::pipeline_depth(self.device.soc()),
+            self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);};
+        // Yield at the next retirement when more urgent work is already waiting.
+        if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency),
+            last.entries[0].0.urgency) {return Ok(None);}
+        if !policy::within_budget(self.pass_cost.get(packet.vm.id()),
+            crate::m3::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);}
+        if !policy::overlap_ready(self.render_initialized,true,true,true,
+            self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);}
+        Ok(Some(room))
+    }
+    /// Publish pending batches while allowed: one when none is active, more behind it with
+    /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed.
+    fn publish_next(&mut self)->bool {
+        let mut progress=false;
+        while !self.pending.is_empty() && self.state.healthy() {
+            let overlap=!self.active.is_empty();
+            let (base,max)=match self.pipeline_slot() {
+                Ok(Some(slot))=>slot,
+                Ok(None)=>break,
+                Err(error)=>{self.fail_active(error);return true;}
+            };
+            let entries=match self.next_batch(max) {
+                Ok(entries) if !entries.is_empty()=>entries,
+                _=>{self.fail_front(ENOMEM);progress=true;continue;},
+            };
+            match self.publish(&entries,base,overlap) {
+                Ok(batch)=>{
+                    // Consume the published commands from `pending`.
+                    let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;};
+                    while let Some((packet,_))=self.pending.first() {
+                        if Arc::ptr_eq(packet,&last) {
+                            if last_index+1==last.commands.len() {let _=self.pending.remove(0);}
+                            else {self.pending[0].1=last_index+1;}
+                            break;
+                        }
+                        let _=self.pending.remove(0);
+                    }
+                    if self.active.push(batch,GFP_KERNEL).is_err() {
+                        // Published but untracked: it can never be retired.
+                        self.capture_fault(ENOMEM);Self::finish_all(&entries,Err(ENOMEM));
+                        self.fail_active(ENOMEM);return true;
+                    }
+                    progress=true;
+                }
+                Err((error,published))=>{
+                    if published {
+                        if crate::m3::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict {
+                            crate::m3::t8122_start::job_setup_failed_verdict(self.drm.as_ref().as_ref(),error);
+                            self.t8122_verdict=true;
+                        }
+                        // A live ring WRITE or doorbell may have been published: keep packets
+                        // retained (self.packets) and fail them and everything in flight.
+                        self.capture_fault(error);
+                        Self::finish_all(&entries,Err(error));
+                        self.fail_active(error);
+                        self.fail_pending(error);
+                        return true;
+                    } else if overlap && error==EBUSY {
+                        // Not publishable behind the running batch (new VM binding or storage
+                        // growth): wait for it to retire.
+                        break;
+                    } else {
+                        // Nothing reached the firmware: fail only the front packet, as a failed
+                        // execute() did before publication.
+                        self.fail_front(error);
+                        progress=true;
+                    }
+                }
+            }
+        }
+        progress
+    }
+    /// Prepare and publish one batch. On error, reports whether firmware may see its writes.
+    fn publish(&mut self,entries:&[(Arc<crate::m3::m3_submit::Packet>,usize)],base:usize,overlap:bool)->core::result::Result<Batch,(Error,bool)> {
+        let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Render{..}));
+        let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Compute(_)));
+        if render && compute {return self.publish_mixed(entries,base,overlap);}
+        let early=|e:Error|(e,false);
+        let (packet,first_index)=(&entries[0].0,entries[0].1);
+        let control=packet.commands[first_index];
+        let batch_count=entries.len();
+        let prepare=Instant::<Monotonic>::now();
+        // All host bookkeeping is reserved before mutating shared producer state.
+        // No allocation may fail after publication and lose a DMA owner.
+        self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        self.packets.reserve(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        let mut owned=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        for (p,i) in entries {owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?;}
+
+        // Retain the batch's packets (and their VM job guards) until it retires; with batches
+        // in flight, also those of the earlier ones.
+        if !overlap {self.packets.clear();}
+        for (p,_) in entries {
+            if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;}
+        }
+        // Before any producer state changes: an append the overlapping firmware consumes, or a
+        // partial publication, makes these packets visible to it.
+        for (p,_) in entries {p.mark_published();}
+        let found=self.jobs.iter().position(|job| matches!((job,control),
+            (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) |
+            (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..})));
+        let index=found.unwrap_or(self.jobs.len());
+        if found.is_none() {
+            let job=match control {
+                crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?),
+                crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.render_stats().map_err(early)?,self.device.soc().clusters,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?),
+            };
+            self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        } else {
+            match (&mut self.jobs[index],control) {
+                (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..})=>{},
+                (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?,
+                _=>return Err(early(ENOTSUPP)),
+            }
+        }
+        if let NativeJob::Compute(j)=&mut self.jobs[index] {
+            let prepared=(||->Result {
+                for (slot,(p,next)) in entries.iter().enumerate() {
+                    if slot!=0 {
+                        let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);};
+                        j.append(c)?;
+                    }
+                    let timestamps=p.timestamp_addresses(*next);
+                    let coalesce=crate::m3::m3_compute_storage::coalesce_stamp_flush(
+                        p.wide_visibility[*next],timestamps,slot+1==batch_count);
+                    j.prepare_completion(coalesce)?;
+                    j.set_user_timestamps(timestamps)?;
+                    j.set_attachments(&p.attachments[*next])?;
+                }
+                Ok(())
+            })();
+            if let Err(error)=prepared {
+                j.abort_batch().map_err(|e|(e,true))?;
+                if found.is_none() {drop(self.jobs.remove(index).map_err(|_|early(EIO))?);}
+                return Err(early(error));
+            }
+        }
+        let mut coalesced_stamps=0;
+        let coalesce_render=crate::m3::m3_params::render_stamp_coalesce();
+        let internal_timestamps=crate::m3::m3_params::render_internal_timestamps();
+        let chip=self.device.soc().chip_id;
+        if let NativeJob::Render(j)=&mut self.jobs[index] {
+            let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+            for (p,next) in entries {commands.push(p.commands[*next],GFP_KERNEL).map_err(|_|early(ENOMEM))?;}
+            j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?;
+            let prepared=(||->Result {
+                for (slot,(p,next)) in entries.iter().enumerate() {
+                    let crate::m3::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);};
+                    let timestamps=p.render_timestamp_addresses(*next);
+                    let coalesce=policy::coalesce_render_stamps(coalesce_render,chip,
+                        overlap,slot,batch_count,timestamps);
+                    let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair));
+                    j.append(command,usc,coalesce,timed)?;
+                    j.set_user_timestamps(timestamps)?;
+                    j.set_attachments(&p.render_attachments[*next])?;
+                    coalesced_stamps+=u64::from(coalesce);
+                }
+                Ok(())
+            })();
+            if let Err(error)=prepared {
+                // Running firmware can consume append's WRITE updates without a new doorbell.
+                // Even an append that returns an error may have changed shared counters or one
+                // pipe. Do not rewind live producer state or let the caller release owners and
+                // reuse these slots. Its published-error path stops scheduling and retains them.
+                if overlap {return Err((error,true));}
+                j.abort_batch().map_err(|e|(e,true))?;
+                return Err(early(error));
+            }
+        }
+        let preparation_ns=prepare.elapsed().as_nanos();
+        // The batch's kernel objects may have created leaf tables in the host window: link
+        // them into the firmware's shared L2 before it is told about the batch. A failure
+        // means the window no longer matches what the firmware may walk: stop scheduling and
+        // retain the batch, as for a published error.
+        if let Err(error)=self.window.sync_all(&self.uat) {
+            return Err((error,true));
+        }
+        // The command aliases live in the context-0 lower tree, which has no bind slot on the
+        // adopted UAT (7.1.12 gave it the fixed slot 0), so remapping one invalidates nothing:
+        // drop context 0's non-global entries before the firmware fetches the new ones.
+        crate::mem::sync();
+        crate::mem::tlbi_asid(0);
+        crate::mem::sync();
+        self.t8122_verdict=false;
+        let previous_events=self.config.completed_events;
+        if self.device.firmware().profile() == super::m3_profile::Profile::Extended {
+            packet.vm.mark_binding_published().map_err(early)?;
+        }
+        let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0};
+        self.state.health.set_gpu_pending(true);
+        self.gpu_pending = true;
+        let late=|e:Error|(e,true);
+        let expected_events=match &self.jobs[index] {
+            NativeJob::Compute(j)=>{
+                self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?;
+                Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?;1
+            },
+            NativeJob::Render(j)=>{
+                if j.first() {self.config.render_pb().map_err(late)?;}
+                let queues=j.queues();
+                self.config.submit_queue(1,queues[1],j.heads()[1],1,j.first()).map_err(late)?;
+                Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?;
+                self.config.submit_queue(0,queues[0],j.heads()[0],0,j.first()).map_err(late)?;
+                Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?;2
+            },
+        };
+        let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary);
+        let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)};
+        Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3::m3_submit::Command::Compute(_))),
+            first:control, preparation_ns, previous_events, expected_events,
+            start:Instant::<Monotonic>::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0},
+            polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None })
+    }
+    /// Prepare both idle queues from one packet. Cross-engine changes wait on
+    /// the preceding command's firmware stamp; all commands remain GPU ordered.
+    /// Producer storage and DMA guards survive every partial publication error.
+    fn publish_mixed(&mut self,entries:&[(Arc<crate::m3::m3_submit::Packet>,usize)],
+        base:usize,overlap:bool)->core::result::Result<Batch,(Error,bool)> {
+        let early=|e:Error|(e,false);
+        if overlap || base!=0 || self.device.soc().chip_id!=0x6030
+            || !self.render_initialized || entries.len()>policy::SLOTS {return Err(early(EINVAL));}
+        let packet=&entries[0].0;
+        if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));}
+        let prepare=Instant::<Monotonic>::now();
+        let internal_timestamps=crate::m3::m3_params::render_internal_timestamps();
+        self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        let mut owned=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        let mut render_commands=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        let mut first_compute=None;
+        let mut computes=0;
+        for (p,i) in entries {
+            owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+            match p.commands[*i] {
+                c@crate::m3::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?,
+                crate::m3::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);},
+            }
+        }
+        let renders=render_commands.len();
+        if renders==0 || computes==0 {return Err(early(EINVAL));}
+        let index=self.jobs.iter().position(|j|matches!(j,NativeJob::Render(r) if r.started())).ok_or(early(EINVAL))?;
+        let existing=self.jobs.iter().position(|j|matches!(j,NativeJob::Compute(_)));
+        let compute_index=existing.unwrap_or(self.jobs.len());
+        // Keep the packet before any producer mutation. No other batch is live.
+        self.packets.clear();
+        self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;
+        packet.mark_published();
+        let mut render_prepared=false;
+        let mut compute_prepared=false;
+        let prepared=(||->Result<usize> {
+            let control=first_compute.ok_or(EINVAL)?;
+            if existing.is_none() {
+                let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,
+                    self.config.stats_region()?,control,self.device.soc().registers,self.device.firmware().profile())?;
+                self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?;
+            } else {
+                let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);};
+                j.replay(&self.uat,&packet.vm,control)?;
+            }
+            compute_prepared=true;
+            if let NativeJob::Compute(j)=&mut self.jobs[compute_index] {
+                j.reserve_barriers(&self.drm,&self.uat)?;
+            }
+            let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);};
+            j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?;
+            render_prepared=true;
+            let mut previous:Option<(bool,crate::m3::m3_sync_layout::CompletionDependency)>=None;
+            let mut compute_slot=0;
+            let mut dependencies=0;
+            for (p,i) in entries {
+                match p.commands[*i] {
+                    crate::m3::m3_submit::Command::Render{command,usc}=>{
+                        let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d);
+                        let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);};
+                        let timestamps=p.render_timestamp_addresses(*i);
+                        let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair));
+                        j.append_after(command,usc,false,predecessor,timed)?;
+                        j.set_user_timestamps(timestamps)?;
+                        j.set_attachments(&p.render_attachments[*i])?;
+                        dependencies+=usize::from(predecessor.is_some());
+                        previous=Some((true,j.completion_dependency()?));
+                    },
+                    crate::m3::m3_submit::Command::Compute(c)=>{
+                        let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);};
+                        if compute_slot!=0 {j.append(c)?;}
+                        if let Some((true,dependency))=previous {
+                            j.insert_dependency(dependency)?;
+                            dependencies+=1;
+                        }
+                        // Every cross-engine predecessor flushes its stamp.
+                        // Final queue, timestamp and cache visibility are unchanged.
+                        j.prepare_completion(false)?;
+                        j.set_user_timestamps(p.timestamp_addresses(*i))?;
+                        j.set_attachments(&p.attachments[*i])?;
+                        previous=Some((false,j.completion_dependency()?));
+                        compute_slot+=1;
+                    },
+                }
+            }
+            Ok(dependencies)
+        })();
+        let dependencies=match prepared {
+            Ok(n)=>n,
+            Err(error)=>{
+                if render_prepared {
+                    let NativeJob::Render(j)=&mut self.jobs[index] else {return Err((EIO,true));};
+                    j.abort_batch().map_err(|e|(e,true))?;
+                }
+                if compute_prepared {
+                    if existing.is_none() {drop(self.jobs.remove(compute_index).map_err(|_|(EIO,true))?);}
+                    else if let NativeJob::Compute(j)=&mut self.jobs[compute_index] {
+                        j.abort_batch().map_err(|e|(e,true))?;
+                    }
+                }
+                return Err(early(error));
+            },
+        };
+        let preparation_ns=prepare.elapsed().as_nanos();
+        let previous_events=self.config.completed_events;
+        let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0};
+        self.state.health.set_gpu_pending(true);self.gpu_pending=true;
+        let late=|e:Error|(e,true);
+        let NativeJob::Render(j)=&self.jobs[index] else {return Err(late(EIO));};
+        let queues=j.queues();let heads=j.heads();let (_,_,last_draw)=j.batch();
+        self.config.submit_queue(1,queues[1],heads[1],1,false).map_err(late)?;
+        Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?;
+        self.config.submit_queue(0,queues[0],heads[0],0,false).map_err(late)?;
+        Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?;
+        let NativeJob::Compute(j)=&self.jobs[compute_index] else {return Err(late(EIO));};
+        self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?;
+        Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?;
+        Ok(Batch {entries:owned,index,kind:2,first:packet.commands[entries[0].1],
+            preparation_ns,previous_events,expected_events:3,start:Instant::<Monotonic>::now(),
+            measure:crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary),
+            doorbell_counter_ns:0,polling_ns:0,polls:0,stamp_ns:-1,first_event_ns:-1,
+            events_ns:-1,completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0,coalesced_stamps:0,base:0,last_draw,publication_tick,
+            mixed:Some(MixedBatch {compute_index,renders,computes,dependencies})})
+    }
+    /// A mixed publication retires only after both jobs, all three queue
+    /// messages, firmware health and engine idle passed the ordinary checks.
+    /// Its wall time is reported once rather than charged to both engines.
+    fn retire_mixed(&mut self,batch:&Batch,mixed:&MixedBatch)->Result {
+        let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);};
+        let render_ns=j.gpu_ns(0,mixed.renders)?;
+        let stages=j.stage_ns(0,mixed.renders)?;
+        j.progress(&self.drm)?;
+        let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);};
+        let compute_ns=j.gpu_ns()?;
+        j.progress(&self.drm)?;
+        if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {
+            let vm=batch.entries[0].0.vm.id();
+            let observed_tick=physical_counter();
+            let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);};
+            let render_span=j.gpu_ticks(0,mixed.renders)?;
+            let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);};
+            let compute_span=j.batch_gpu_span()?;
+            let start=render_span.0.min(compute_span[0]);
+            let end=render_span.1.max(compute_span[1]);
+            let valid=batch.publication_tick!=0 && start>=batch.publication_tick && end>=start && observed_tick>=end;
+            dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} mixed=1 batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n",
+                vm,batch.entries.len(),batch.publication_tick,start,end,observed_tick,u32::from(valid));
+            self.log_batch_milestones(batch,render_span.0,render_span.1,render_ns,stages[1]);
+            // Render and compute have separate slot sequences in one mixed batch.
+            let mut render_slot=0;
+            let mut compute_slot=0;
+            for (packet,next) in batch.entries.iter() {
+                match packet.commands[*next] {
+                    crate::m3::m3_submit::Command::Render{command:r,..}=>{
+                        let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);};
+                        let s=j.slot_stage_ns(0,mixed.renders,render_slot)?;
+                        dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",
+                            vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]);
+                        render_slot+=1;
+                    },
+                    crate::m3::m3_submit::Command::Compute(c)=>{
+                        let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);};
+                        let ns=j.slot_gpu_ns(compute_slot)?;
+                        dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",
+                            vm,compute_slot,mixed.computes,c.base,c.end,c.usc_base,ns);
+                        compute_slot+=1;
+                    },
+                }
+            }
+        }
+        self.state.health.record_completion();
+        self.render_batches[mixed.renders-1]+=1;
+        self.pass_cost.record(batch.entries[0].0.vm.id(),render_ns/mixed.renders as u64);
+        self.mixed_batches+=1;
+        if (self.mixed_batches==1 || self.mixed_batches%128==0) && batch.measure {
+            dev_info!(self.drm.as_ref(),"M3_MIXED_BATCH_RETIRED count={} commands={} render={} compute={} barriers={} prepare_ns={} active_ns={} render_gpu_ns={} compute_gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} stamps=1 queues=1 events=3 idle=1\n",
+                self.mixed_batches,batch.entries.len(),mixed.renders,mixed.computes,mixed.dependencies,
+                batch.preparation_ns,batch.start.elapsed().as_nanos(),render_ns,compute_ns,stages[0],stages[1],stages[2]);
+        }
+        if self.active.is_empty() {
+            self.gpu_pending=false;self.state.health.set_gpu_pending(false);
+        }
+        Ok(())
+    }
+    /// Check the active batch once. Ok(true): retired (its packets are signalled).
+    fn poll_active(&mut self)->Result<bool> {
+        let Some((index,measure,last_draw))=self.active.first().map(|b|(b.index,b.measure,b.last_draw)) else {return Ok(false);};
+        let poll_start=measure.then(Instant::<Monotonic>::now);
+        if !self.state.healthy() || self.config.drain(&self.drm).is_err() {
+            self.fail_active(EIO);return Err(EIO);
+        }
+        // With later batches in flight, the oldest retires once its own last draw's stamps
+        // have arrived; the last one is also checked for exact queue, event and engine idle.
+        if self.active.len()>1 {
+            let done=match &mut self.jobs[index] {NativeJob::Render(j)=>j.complete_through(last_draw),_=>Err(EINVAL)};
+            match done {
+                Ok(true)=>{
+                    let batch=self.active.remove(0).map_err(|_|EIO)?;
+                    if let Err(e)=self.retire(&batch) {
+                        self.capture_fault(e);Self::finish_all(&batch.entries,Err(e));
+                        self.fail_active(e);return Err(e);
+                    }
+                    self.retire_packets(&batch.entries);
+                    return Ok(true);
+                }
+                Ok(false)=>{
+                    let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?;
+                    if expired {
+                        dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous);
+                        self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT);
+                    }
+                    return Ok(false);
+                }
+                Err(e)=>{self.fail_active(e);return Err(e);}
+            }
+        }
+        let secondary=self.active.first().and_then(|b|b.mixed.as_ref().map(|m|m.compute_index));
+        let completion=(||->Result<bool> {
+            let primary=self.jobs[index].complete()?;
+            let other=match secondary {Some(i)=>self.jobs[i].complete()?,None=>true};
+            Ok(primary && other)
+        })();
+        let stamped=match completion {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}};
+        if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {
+            let before=physical_counter();
+            let parts=match &mut self.jobs[index] {
+                NativeJob::Render(j)=>j.fragment_stamp_parts().ok(),
+                _=>None,
+            };
+            let after=physical_counter();
+            if let Some(parts)=parts {
+                if let Some(batch)=self.active.first_mut() {
+                    if batch.fragment_host_tick==0 {
+                        if parts[0] {batch.fragment_host_tick=after;}
+                        else {batch.fragment_host_pending_tick=before;}
+                    }
+                    if batch.fragment_fw_tick==0 {
+                        if parts[1] {batch.fragment_fw_tick=after;}
+                        else {batch.fragment_fw_pending_tick=before;}
+                    }
+                }
+            }
+        }
+        let power_enabled=crate::m3::m3_params::active_power();
+        if power_enabled {
+            let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos());
+            if crate::m3::m3_active_power::due(power_enabled,self.device.soc().chip_id,
+                                          stamped,self.active_power_samples,since_last) {
+                self.active_power_last=Some(Instant::<Monotonic>::now());
+                self.active_power_samples+=1;
+                // Diagnostic read errors do not change job completion or recovery.
+                if let Err(e)=self.config.log_active_power(&self.drm,&self.device,
+                                                          self.active_power_samples) {
+                    dev_info!(self.drm.as_ref(),"M3_ACTIVE_POWER sample={} unavailable={:?}\n",
+                              self.active_power_samples,e);
+                }
+            }
+        }
+        let completion_parts=if measure && secondary.is_none() {
+            match &mut self.jobs[index] {
+                NativeJob::Render(j)=>Some(if stamped {[true;4]} else {
+                    match j.completion_parts() {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}}
+                }),
+                _=>None,
+            }
+        } else {None};
+        let completed=self.config.completed_events;
+        let (evented,expired,previous)={
+            let batch=self.active.first_mut().ok_or(EIO)?;
+            let evented=completed>=batch.previous_events+batch.expected_events;
+            if batch.measure {
+                let at=batch.start.elapsed().as_nanos();
+                if stamped && batch.stamp_ns<0 {batch.stamp_ns=at;}
+                if let Some(parts)=completion_parts {
+                    for (part,done) in parts.iter().enumerate() {
+                        if *done && batch.completion_ns[part]<0 {batch.completion_ns[part]=at;}
+                    }
+                }
+                if completed>batch.previous_events && batch.first_event_ns<0 {batch.first_event_ns=at;}
+                if evented && batch.events_ns<0 {batch.events_ns=at;}
+            }
+            if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;}
+            (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events)
+        };
+        if stamped && evented {
+            let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc());
+            match self.device.check_idle_parts(
+                retire_mmio&crate::m3::m3_params::RETIRE_MMIO_BUSY!=0,
+                retire_mmio&crate::m3::m3_params::RETIRE_MMIO_FAULTS!=0) {
+                Ok(())=>match self.config.pipes_idle() {
+                    Ok(true)=>{
+                        if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {
+                            let tick=physical_counter();
+                            if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;}
+                        }
+                        if retire_mmio&crate::m3::m3_params::RETIRE_MMIO_PSTATE!=0 {
+                            if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") {
+                                if crate::m3::t8122_start::is_t8122(self.device.soc()) {
+                                    crate::m3::t8122_start::cap_violated_verdict(self.drm.as_ref().as_ref(),e);
+                                    self.t8122_verdict=true;
+                                }
+                                self.fail_active(e);return Err(e);
+                            }
+                        }
+                        // Stamps, both queue indices, required events,
+                        // firmware health, engines and pipes are verified.
+                        let batch=self.active.remove(0).map_err(|_|EIO)?;
+                        if let Err(e)=self.retire(&batch) {
+                            self.capture_fault(e);
+                            Self::finish_all(&batch.entries,Err(e));
+                            self.fail_pending(e);
+                            return Err(e);
+                        }
+                        // Signal every packet whose last command was in this batch. A packet
+                        // that continues with other commands is still at the front of `pending`.
+                        self.retire_packets(&batch.entries);
+                        return Ok(true);
+                    },
+                    Ok(false)=>{}, // Firmware must consume every submitted queue message.
+                    Err(e)=>{self.fail_active(e);return Err(e);}
+                },
+                Err(e) if e==EBUSY=>{}, // Retirement can trail the event.
+                Err(e)=>{self.fail_active(e);return Err(e);}
+            }
+        }
+        if expired {
+            dev_err!(self.drm.as_ref(),"M3 completion events={} previous={}\n",completed,previous);
+            self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT);
+        }
+        Ok(false)
+    }
+    fn log_batch_milestones(&self,batch:&Batch,render_start:u64,render_end:u64,
+        render_gpu_ns:u64,fragment_sum_ns:u64) {
+        dev_info!(self.drm.as_ref(),"M3_BATCH_MILESTONE vm={} publication_tick={} draw={} mixed={} render_start_tick={} render_end_tick={} render_gpu_ns={} fragment_sum_ns={} fragment_host_tick={} fragment_fw_tick={} fragment_host_pending_tick={} fragment_fw_pending_tick={} pipes_idle_tick={} counter_hz=24000000\n",
+            batch.entries[0].0.vm.id(),batch.publication_tick,batch.last_draw,u32::from(batch.mixed.is_some()),
+            render_start,render_end,render_gpu_ns,fragment_sum_ns,batch.fragment_host_tick,batch.fragment_fw_tick,
+            batch.fragment_host_pending_tick,batch.fragment_fw_pending_tick,batch.pipes_idle_tick);
+        for (packet,command) in batch.entries.iter() {
+            dev_info!(self.drm.as_ref(),"M3_BATCH_PACKET vm={} publication_tick={} packet={:#x} command={}\n",
+                packet.vm.id(),batch.publication_tick,packet.profile_id(),command);
+        }
+    }
+    /// Bookkeeping for a retired batch, then signal its completed packets.
+    fn retire(&mut self,batch:&Batch)->Result {
+        if let Some(mixed)=&batch.mixed {return self.retire_mixed(batch,mixed);}
+        let retire_start=batch.measure.then(Instant::<Monotonic>::now);
+        let index=batch.index;let batch_count=batch.entries.len();let control=batch.first;
+        self.state.health.record_completion();
+        if self.active.is_empty() {
+            match &mut self.jobs[index] {
+                NativeJob::Compute(j)=>j.progress(&self.drm)?,
+                NativeJob::Render(j)=>j.progress(&self.drm)?,
+            }
+        }
+        let retired_counter_ns=if batch.measure {physical_counter() as i64 * 1000 / 24} else {0};
+        let ticks=match &mut self.jobs[index] {
+            NativeJob::Render(j) if batch.measure=>Some(j.gpu_ticks(batch.base,batch_count)?),
+            _=>None,
+        };
+        let stages=match &mut self.jobs[index] {
+            NativeJob::Render(j)=>j.stage_ns(batch.base,batch_count)?,
+            _=>[0;3],
+        };
+        let (kind,gpu_ns)=match &mut self.jobs[index] {
+            NativeJob::Compute(j)=>(1,j.gpu_ns()?),
+            NativeJob::Render(j)=>(0,j.gpu_ns(batch.base,batch_count)?),
+        };
+        if kind==0 {
+            self.render_batches[batch_count-1]+=1;
+            if batch.measure && batch.coalesced_stamps>0 && self.coalesced_render_stamps==0 {
+                if let NativeJob::Render(j)=&mut self.jobs[index] {
+                    let masks=j.stamp_flush_masks(batch.base,batch_count)?;
+                    dev_info!(self.drm.as_ref(),"M3_RENDER_STAMP_FLUSH vm={} batch={} ta_mask={:#x} fs_mask={:#x} coalesced={}\n",
+                        batch.entries[0].0.vm.id(),batch_count,masks[0],masks[1],batch.coalesced_stamps);
+                }
+            }
+            self.coalesced_render_stamps+=batch.coalesced_stamps;
+            let vm=batch.entries[0].0.vm.id();
+            self.pass_cost.record(vm,gpu_ns/batch_count as u64);
+            self.render_initialized=true;
+        }
+        if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {
+            let observed_tick=physical_counter();
+            let (cs,span)=match (&mut self.jobs[index],control) {
+                (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?),
+                (NativeJob::Render(j),crate::m3::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])},
+                _=>return Err(EINVAL),
+            };
+            let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1];
+            dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} cs={:#x} batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n",
+                batch.entries[0].0.vm.id(),cs,batch_count,batch.publication_tick,span[0],span[1],observed_tick,u32::from(valid));
+            self.log_batch_milestones(batch,
+                if kind==0 {span[0]} else {0},if kind==0 {span[1]} else {0},
+                if kind==0 {gpu_ns} else {0},stages[1]);
+            if let NativeJob::Compute(j)=&mut self.jobs[index] {
+                for (slot,(p,next)) in batch.entries.iter().enumerate() {
+                    let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);};
+                    let ns=j.slot_gpu_ns(slot)?;
+                    dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns);
+                }
+            }
+            if let NativeJob::Render(j)=&mut self.jobs[index] {
+                for (slot,(p,next)) in batch.entries.iter().enumerate() {
+                    let crate::m3::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);};
+                    let s=j.slot_stage_ns(batch.base,batch_count,slot)?;
+                    dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]);
+                }
+            }
+        }
+        let preparation_ns=batch.preparation_ns;
+        let active_ns=batch.start.elapsed().as_nanos();
+        self.state.health.timing.record(kind==0, batch_count as u64,
+            preparation_ns, active_ns, gpu_ns, stages);
+        let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos());
+        if batch.measure {
+            if let crate::m3::m3_submit::Command::Render {command:r,..}=control {
+                if batch_count>1 {self.geometry_overflow+=1;} else {
+                if let Some(g)=self.geometry.iter_mut().find(|g|
+                    g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) {
+                    g.width=r.width;g.height=r.height;g.samples=r.samples;
+                    g.count+=1;g.prepare_ns+=preparation_ns;g.active_ns+=active_ns;
+                    g.gpu_ns+=gpu_ns;g.tiling_ns+=stages[0];g.fragment_ns+=stages[1];
+                    g.max_fragment_ns=g.max_fragment_ns.max(stages[1]);
+                    g.polling_ns+=batch.polling_ns;g.retirement_ns+=retirement_ns;
+                    g.polls+=batch.polls;
+                    g.stamp_ns+=batch.stamp_ns;g.first_event_ns+=batch.first_event_ns;g.events_ns+=batch.events_ns;
+                    for (total,observed) in g.completion_ns.iter_mut().zip(batch.completion_ns) {*total+=observed;}
+                    if let Some((ta,fe))=ticks {
+                        g.pickup_ns+=(ta as i64)*1000/24-batch.doorbell_counter_ns;
+                        g.tail_ns+=retired_counter_ns-(fe as i64)*1000/24;
+                    }
+                } else {
+                    self.geometry_overflow+=1;
+                }
+                }
+            }
+        }
+        if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            match control {
+                crate::m3::m3_submit::Command::Render {command:r,..}=>
+                    dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]),
+                crate::m3::m3_submit::Command::Compute(_)=>
+                    dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns),
+            }
+        }
+        let t=&mut self.timing[kind];
+        t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64;
+        for i in 0..3 {t[4+i]+=stages[i] as i64;}
+        // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic.
+        if t[0]==1 && crate::m3::t8122_start::is_t8122(self.device.soc()) {
+            let span=match &mut self.jobs[index] {
+                NativeJob::Compute(j)=>j.batch_gpu_span().ok(),
+                NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]),
+            };
+            crate::m3::t8122_start::job_completed_verdict(self.drm.as_ref().as_ref(),kind,gpu_ns,span);
+        }
+        if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) {
+            dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6],
+                match &self.jobs[index] {NativeJob::Render(j)=>j.ordinal(),NativeJob::Compute(j)=>j.ordinal()});
+        }
+        if kind==0 && t[0]%512==0 &&
+            crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary) {
+            let early=match &self.jobs[index] {NativeJob::Render(j)=>j.early_count(),_=>0};
+            dev_info!(self.drm.as_ref(),"M3_ORDERED_BATCHES depths={:?} early={} packets={} coalesced_stamps={}\n", self.render_batches,early,self.coalesced_packets,self.coalesced_render_stamps);
+            let now=<Monotonic as kernel::time::ClockSource>::ktime_get();
+            dev_info!(self.drm.as_ref(),"M3_GEOMETRY_BATCH ns={} render_count={} overflow={}\n",
+                now,t[0],self.geometry_overflow);
+            self.geometry_overflow=0;
+            for g in &mut self.geometry {
+                if g.count==0 {continue;}
+                dev_info!(self.drm.as_ref(),"M3_GEOMETRY ns={} width={} height={} samples={} count={} prepare_ns={} active_ns={} gpu_ns={} tiling_ns={} fragment_ns={} max_fragment_ns={} polling_ns={} retirement_ns={} sleeping_ns={} polls={} stamp_ns={} first_event_ns={} events_ns={} pickup_ns={} tail_ns={} gpu_stamp_ns={} fw_stamp_ns={} queue_done_ns={} queue_read_ns={}\n",
+                    now,g.width,g.height,g.samples,g.count,g.prepare_ns,g.active_ns,g.gpu_ns,g.tiling_ns,g.fragment_ns,g.max_fragment_ns,g.polling_ns,g.retirement_ns,g.sleeping_ns,g.polls,g.stamp_ns,g.first_event_ns,g.events_ns,g.pickup_ns,g.tail_ns,g.completion_ns[0],g.completion_ns[1],g.completion_ns[2],g.completion_ns[3]);
+                *g=GeometryTiming::default();
+            }
+        }
+        if self.active.is_empty() {
+            self.gpu_pending = false;
+            self.state.health.set_gpu_pending(false);
+        }
+        for pair in batch.entries.windows(2) {
+            if !Arc::ptr_eq(&pair[0].0,&pair[1].0) {self.coalesced_packets+=1;}
+        }
+        Ok(())
+    }
+}
+// Keep the large graph on the heap: returning and moving it through probe,
+// Registration and Mutex initialization otherwise duplicates it on the bounded
+// kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop.
+pub(crate) struct Runtime { inner: ManuallyDrop<KBox<Inner>> }
+/// Claimed before this boot's first ASC start.
+static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new();
+/// Takes a module reference, unless the module is already being removed.
+#[cfg(CONFIG_MODULE_UNLOAD)]
+fn pin_module() -> bool {
+    // SAFETY: THIS_MODULE is this module, whose code is running; try_module_get fails once
+    // removal has begun instead of reviving the reference count.
+    unsafe { kernel::bindings::try_module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr()) }
+}
+/// Without module unloading there is no removal to race.
+#[cfg(not(CONFIG_MODULE_UNLOAD))]
+fn pin_module() -> bool { true }
+/// Drops a reference [`pin_module`] took.
+#[cfg(CONFIG_MODULE_UNLOAD)]
+fn unpin_module() {
+    // SAFETY: called only to drop a reference pin_module took.
+    unsafe { kernel::bindings::module_put(kernel::module::this_module::<crate::LocalModule>().as_ptr()) };
+}
+#[cfg(not(CONFIG_MODULE_UNLOAD))]
+fn unpin_module() {}
+impl Runtime {
+    #[inline(never)]
+    pub(crate) fn new(pdev: &platform::Device<Core<'_>>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result<Self> {
+        // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit
+        // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does
+        // not borrow `device`, which is moved into Inner below.
+        let t8122 = crate::m3::t8122_start::is_t8122(device.soc());
+        let coproc_refused = |e: &Error| {
+            if t8122 { crate::m3::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); }
+        };
+        device.require_stopped(pdev).inspect_err(coproc_refused)?;
+        // Reserve before starting ASC. ENOMEM must not drop firmware owners
+        // while the coprocessor may still access them. Writing the completed
+        // graph into this allocation is infallible.
+        let owner = KBox::<Inner>::new_uninit(GFP_KERNEL)?;
+        let drm: driver::AsahiDevRef = drm.into();
+        let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?;
+        let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?;
+        if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) {
+            return Err(ENODEV);
+        }
+        // A second start (unbind and rebind, a module reload, a retry after a failed start)
+        // would set RUN while TTBAT context 0 still names the freed root of this one. An ordinary
+        // (not forced) unload is blocked once the latch is claimed, so a reload can't forget it.
+        // The pin comes first: a later probe can race an ordinary rmmod, and once removal has
+        // begun nothing starts.
+        match policy::claim_start(&FIRMWARE_START, pin_module, unpin_module) {
+            Ok(()) => {}
+            Err(policy::StartRefusal::Removing) => {
+                dev_err!(pdev.as_ref(), "M3: the module is being removed; GPU not started\n");
+                return Err(ENODEV);
+            }
+            Err(policy::StartRefusal::AlreadyStarted) => {
+                dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n");
+                return Err(EBUSY);
+            }
+        }
+        // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict.
+        let mut started=false;
+        // The adopted UAT and its window live outside the closure: once TTBAT context 0 names
+        // the kernel-owned lower root, a failure must not free those tables before ASC stops.
+        let mut adopted=None;
+        let prepared=(|| -> Result<_> {
+            device.start_asc(pdev)?;
+            Pin::new(&mut transport).wake()?;
+            if !state.healthy() || !Pin::new(&transport).is_running() {return Err(EIO);}
+            for ep in [0x20,0x21] {
+                if !Pin::new(&mut transport).has_endpoint(ep) {return Err(ENODEV);}
+                Pin::new(&mut transport).start_endpoint(ep)?;
+            }
+            started=true;
+            // The firmware's UAT, adopted as the G15 backend adopts it on 7.2: the handoff and
+            // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as
+            // every context's upper half, and the host window (root entry 2) through the
+            // shared L2. The RTKit is awake and no initdata or GPU job has been published.
+            let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources,device.firmware().profile())?);
+            let config=Config::new(&drm,uat,device.firmware(),&contents)?;
+            // Link every kernel table the configuration created before the firmware sees an
+            // address in it, then drop any translation cached before they existed.
+            window.sync_all(uat)?;
+            crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();
+            Ok(config)
+        })();
+        let (uat,window,config)=match (prepared,adopted.take()) {
+            (Ok(config),Some((uat,window)))=>(uat,window,config),
+            (prepared,adopted)=>{
+                let e=prepared.err().unwrap_or(EINVAL);
+                crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e);
+                state.health.mark_failed();
+                if device.stop_asc().is_err() {
+                    // Without module unloading, the C helper is an unbound no-op.
+                    #[cfg(CONFIG_MODULE_UNLOAD)]
+                    unsafe {kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr())};
+                    core::mem::forget(transport);core::mem::forget(state);
+                    core::mem::forget(adopted);
+                    core::mem::forget(drm);core::mem::forget(device);
+                } else {
+                    // ASC has stopped: the firmware no longer walks the adopted tables.
+                    drop(adopted);
+                }
+                return Err(e);
+            }
+        };
+        Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device,
+            jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS],
+            geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false,
+            boot_step:crate::m3::t8122_start::BootStep::Publish })) })
+    }
+    pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() }
+    pub(crate) fn health(&self) -> Arc<m3_rtkit::Health> { self.inner.state.health.clone() }
+    /// The RTKit callbacks' state, which owns the completion worker.
+    pub(crate) fn rtkit_state(&self) -> Arc<m3_rtkit::State> { self.inner.state.clone() }
+    pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range<u64>) -> Result<mmu::Vm> {
+        use crate::util::RangeExt;
+        let reserved_range=0x70_0000_0000..0x80_0000_0000;
+        if range.overlaps(reserved_range.clone()) {return Err(EINVAL);}
+        if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); }
+        // Status and job lifetime tracking: deferred unmaps while accepted work runs
+        // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2).
+        let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?;
+        let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?;
+        let mappings = self.inner.config.global_table_views(&vm)?;
+        vm.install_driver_mappings(mappings,reserved)?;
+        Ok(vm)
+    }
+    pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range<usize>)->Result<mmu::KernelMapping> {
+        if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); }
+        let mapping=bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::m3::agx_memory::TIMESTAMP_RANGE,
+            mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false)?;
+        // A job of an overlapping batch can reach the firmware without a new publication:
+        // link a new leaf table now, not only at the next batch.
+        let inner: &mut Inner=&mut self.inner;
+        inner.window.sync_all(&inner.uat)?;
+        Ok(mapping)
+    }
+    /// Queue a packet and return its fence at once. The scheduler can hand over further
+    /// ready jobs up to its credit limit while the GPU runs. Only the completion worker
+    /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded
+    /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full
+    /// retirement. A burst can therefore share a firmware round trip across packets.
+    pub(crate) fn submit(shared:&crate::m3::m3_drm::Shared,packet:Arc<crate::m3::m3_submit::Packet>)->Result {
+        crate::debug::update_debug_flags();
+        let mut guard=shared.lock();
+        let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner;
+        if !inner.state.healthy() {return Err(EIO);}
+        if packet.commands.is_empty() || packet.commands.len()>256 {
+            pr_err!("M3 submit limit: commands={} maximum=256\n",packet.commands.len());
+            return Err(E2BIG);
+        }
+        // Higher urgency first, FIFO within one urgency, including unfinished packets.
+        // More urgent work can overtake a continuation between retired batches.
+        // Packets of one queue share an urgency, so their order is kept.
+        let urgency=packet.urgency;
+        let at=policy::pending_position(inner.pending.iter().map(|(p,_)|p.urgency),urgency);
+        if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);}
+        inner.pending.reserve(1,GFP_KERNEL)?;
+        inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?;
+        let state=inner.state.clone();
+        drop(guard);
+        crate::m3::m3_completion::queue(&state);
+        Ok(())
+    }
+    /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active
+    /// commands remain owned until retirement; error signaling cannot release a DMA guard.
+    pub(crate) fn withdraw(shared:&crate::m3::m3_drm::Shared,packet:&Arc<crate::m3::m3_submit::Packet>)->bool {
+        let mut guard=shared.lock();
+        let Some(runtime)=Option::as_mut(&mut *guard) else {return false;};
+        let inner: &mut Inner=&mut runtime.inner;
+        let removed=inner.pending.iter().position(|(p,_)|Arc::ptr_eq(p,packet))
+            .is_some_and(|i|inner.pending.remove(i).is_ok());
+        if !inner.packet_active(packet) && !inner.fault_captured {packet.release_guard();}
+        removed
+    }
+    pub(crate) fn completion_wait(&self) -> Arc<m3_rtkit::EventWait> { self.inner.state.events.clone() }
+    pub(crate) fn events(&self)->Arc<m3_rtkit::EventWait> {self.inner.state.events.clone()}
+    /// One completion-worker step: retire the active batch if it has completed, then publish
+    /// the next one. The worker drops the runtime lock between steps.
+    pub(crate) fn service_job(&mut self)->Service {
+        let chip=self.inner.device.soc().chip_id;
+        let busy=self.inner.state.healthy() &&
+            (!self.inner.active.is_empty() || !self.inner.pending.is_empty());
+        if let Err(error)=self.inner.config.performance_request(chip,busy) {
+            self.inner.fail_active(error);
+            return Service::Idle;
+        }
+        let result=self.service_job_inner();
+        if matches!(result,Service::Idle) || !self.inner.state.healthy() ||
+            (self.inner.active.is_empty() && self.inner.pending.is_empty()) {
+            let _=self.inner.config.performance_request(chip,false);
+        }
+        result
+    }
+    fn service_job_inner(&mut self)->Service {
+        let inner: &mut Inner=&mut *self.inner;
+        if inner.active.is_empty() {
+            if inner.fault_captured || !inner.state.healthy() {inner.fail_pending(EIO);return Service::Idle;}
+            if let Err(error)=inner.config.drain(&inner.drm) {
+                inner.capture_fault(error);inner.fail_pending(EIO);return Service::Idle;
+            }
+            return if inner.publish_next() {Service::Progress} else {Service::Idle};
+        }
+        let messages=inner.state.event_messages.load(Ordering::Acquire);
+        match inner.poll_active() {
+            Ok(true)=>{inner.publish_next();Service::Progress},
+            Ok(false)=>if inner.publish_next() {Service::Progress} else {Service::Waiting(messages)},
+            Err(_)=>{inner.fail_pending(EIO);Service::Idle},
+        }
+    }
+    pub(crate) fn boot(&mut self, pdev: &platform::Device<Core<'_>>) -> Result {
+        let is_t8122 = crate::m3::t8122_start::is_t8122(self.inner.device.soc());
+        self.inner.boot_step = crate::m3::t8122_start::BootStep::Publish;
+        let result = self.boot_inner(pdev);
+        if let Err(error) = result {
+            // On T8122, log the firmware readiness words on every boot failure, so the verdict's
+            // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in
+            // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here.
+            let logged = self.inner.boot_step == crate::m3::t8122_start::BootStep::AwaitReady
+                && error == ETIMEDOUT;
+            if is_t8122 && !logged {
+                let inner: &mut Inner = &mut *self.inner;
+                let _ = inner.config.log_ready(&inner.drm);
+            }
+            self.inner.capture_fault(error);
+        }
+        if is_t8122 {
+            let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false);
+            crate::m3::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic,
+                self.inner.boot_step, accepted, self.inner.state.health.crashed(), result);
+        }
+        result
+    }
+    fn boot_inner(&mut self, pdev: &platform::Device<Core<'_>>) -> Result {
+        use crate::m3::t8122_start::BootStep;
+        if self.inner.device.firmware().profile() == super::m3_profile::Profile::Extended {
+            return self.boot_extended(pdev);
+        }
+        let root = self.inner.config.root();
+        dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root);
+        self.inner.boot_step = BootStep::Publish;
+        Pin::new(&mut self.inner.transport).send_message(0x20, 0x0081000000000000 | (root & ((1u64<<44)-1)))?;
+        self.inner.boot_step = BootStep::DeviceControl;
+        self.send_control(0x13)?;
+        self.send_control(9)?;
+        self.inner.boot_step = BootStep::AwaitReady;
+        let start = Instant::<Monotonic>::now();
+        while !self.inner.config.ready()? {
+            if start.elapsed() >= Delta::from_secs(2) || !self.inner.state.healthy() {
+                let inner: &mut Inner=&mut *self.inner;let _=inner.config.log_ready(&inner.drm);
+                return Err(ETIMEDOUT);
+            }
+            fsleep(Delta::from_millis(1));
+        }
+        self.inner.boot_step = BootStep::PostReady;
+        // Import the configured idle policy again after initial power-up.
+        self.send_control(0x13)?;
+        let inner: &mut Inner = &mut *self.inner;
+        inner.config.drain(&inner.drm)?;
+        inner.config.log_ready(&inner.drm)?;
+        if inner.device.soc().chip_id==0x6030 &&
+            (crate::m3::m3_params::active_power() || crate::m3::m3_params::globals_debug_off()) {
+            inner.config.log_globals_debug(&inner.drm)?;
+        }
+        inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?;
+        inner.state.health.set_gpu_pending(false);
+        dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n");
+        Ok(())
+    }
+    fn boot_extended(&mut self, pdev: &platform::Device<Core<'_>>) -> Result {
+        use super::m3_channel_format::Action;
+        use crate::m3::t8122_start::BootStep;
+        // The first control is pending when the root becomes visible. Its
+        // retirement comes from InitData consumption, without a work kick.
+        let first = self.inner.config.control_extended(Action::Initialize)?;
+        self.inner.config.publish_io_table(pdev, self.inner.device.firmware())?;
+        let root = self.inner.config.root();
+        Pin::new(&mut self.inner.transport).send_message(0x20,
+            0x0081_0000_0000_0000 | (root & ((1u64 << 44) - 1)))?;
+        self.inner.boot_step = BootStep::AwaitReady;
+        let deadline = Instant::<Monotonic>::now();
+        while !self.inner.config.ready()? {
+            let inner: &mut Inner = &mut *self.inner;
+            inner.config.drain(&inner.drm)?;
+            if !inner.state.healthy() { return Err(EIO); }
+            if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); }
+            fsleep(Delta::from_millis(1));
+        }
+        self.retire_extended_control(first)?;
+        self.inner.boot_step = BootStep::DeviceControl;
+        // Each data master registers its two firmware engine classes.
+        for ordinal in 0..6 {
+            let action = Action::Register { pipe: ordinal / 2, engine: ordinal % 2 + 1 };
+            self.send_extended_control(action)?;
+        }
+        self.inner.boot_step = BootStep::PostReady;
+        self.send_extended_control(Action::Idle { enabled: true })?;
+        let inner: &mut Inner = &mut *self.inner;
+        inner.config.check_pstate(&inner.drm, &inner.device, "after native boot")?;
+        inner.config.log_ready(&inner.drm)?;
+        inner.state.health.set_gpu_pending(false);
+        dev_info!(pdev.as_ref(), "M3: extended firmware ready; native controllers and idle policy admitted\n");
+        Ok(())
+    }
+
+    fn send_extended_control(&mut self, action: super::m3_channel_format::Action) -> Result {
+        let cursor = self.inner.config.control_extended(action)?;
+        Pin::new(&mut self.inner.transport).send_message(0x21, 0x0084_0000_0000_0011)?;
+        self.retire_extended_control(cursor)
+    }
+
+    fn retire_extended_control(&mut self, expected: u32) -> Result {
+        let deadline = Instant::<Monotonic>::now();
+        loop {
+            let inner: &mut Inner = &mut *self.inner;
+            inner.config.drain(&inner.drm)?;
+            if !inner.state.healthy() { return Err(EIO); }
+            if inner.config.control_done(expected)? { return Ok(()); }
+            if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); }
+            fsleep(Delta::from_millis(1));
+        }
+    }
+
+    fn send_control(&mut self, opcode: u32) -> Result {
+        let next = match self.inner.config.control(opcode) {
+            Ok(next) => next,
+            Err(error) => {
+                dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} queue failed ({:?})\n", opcode, error);
+                return Err(error);
+            }
+        };
+        let send_start = Instant::<Monotonic>::now();
+        if let Err(error) = Pin::new(&mut self.inner.transport).send_message(0x21, 0x0083000000000011) {
+            dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} mailbox send failed ({:?}), elapsed_ns={}\n",
+                opcode, error, send_start.elapsed().as_nanos());
+            return Err(error);
+        }
+        let start = Instant::<Monotonic>::now();
+        loop {
+            let inner: &mut Inner = &mut *self.inner;
+            inner.config.drain(&inner.drm)?;
+            if !inner.state.healthy() { return Err(EIO); }
+            if inner.config.control_done(next)? { return Ok(()); }
+            if start.elapsed() >= Delta::from_secs(2) {
+                dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} acknowledgement timed out, elapsed_ns={}\n",
+                    opcode, start.elapsed().as_nanos());
+                return Err(ETIMEDOUT);
+            }
+            fsleep(Delta::from_millis(1));
+        }
+    }
+}
+impl Drop for Runtime {
+    fn drop(&mut self) {
+        let chip=self.inner.device.soc().chip_id;
+        let _=self.inner.config.performance_request(chip,false);
+        self.inner.state.health.mark_failed();
+        // Unpublished packets never reach the firmware; a published batch's packets stay
+        // retained below if the GPU may still use them, but their fences must not hang.
+        self.inner.fail_pending(ENODEV);
+        for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));}
+        if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) {
+            // SAFETY: a live module reference pins callbacks with retained DMA.
+            #[cfg(CONFIG_MODULE_UNLOAD)]
+            unsafe { kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr()) };
+            return;
+        }
+        // SAFETY: no GPU command is pending and ASC is stopped. Field order
+        // drains callbacks before dropping retained commands, UAT and power.
+        unsafe { ManuallyDrop::drop(&mut self.inner) };
+    }
+}
+
+/// Same scaling contract as G16's qualified publication profiler. AP counter
+/// rates may differ between SoCs; firmware command timestamps use 24 MHz.
+pub(crate) fn physical_counter() -> u64 {
+    let tick: u64;
+    let frequency: u64;
+    // SAFETY: Read architectural counters only; no device state is modified.
+    unsafe { core::arch::asm!("mrs {f}, cntfrq_el0", "mrs {t}, cntpct_el0",
+        f = out(reg) frequency, t = out(reg) tick,
+        options(nomem, nostack, preserves_flags)) };
+    let frequency = frequency & 0xffff_ffff;
+    if frequency == 0 { return 0; }
+    (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency
+}
+
+/// Adopts the running firmware's UAT on `soc`: the `mmu::Uat` of the admitted reserved regions
+/// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the
+/// G15 backend does it on 7.2 (`g15::uat`).
+fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc,
+    resources: &crate::m3::agx_resources::Resources, profile: super::m3_profile::Profile)
+    -> Result<(mmu::Uat, crate::g15::uat::Window)> {
+    let cfg = soc.hwcfg.ok_or(ENODEV)?;
+    let [ttbs, tables, handoff, shared_l2, ..] = resources.regions;
+    let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size;
+    let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas };
+    let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base };
+    let uat = mmu::Uat::new_adopted_with_user_upper(drm, geometry, &regions,
+        profile == super::m3_profile::Profile::Legacy)?;
+    let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?;
+    window.bootstrap(drm.as_ref().as_ref(), &uat)?;
+    // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be
+    // reserved System RAM mapped cacheable.
+    window.clean_tables();
+    Ok((uat, window))
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs
index 880cf4c2b9a75..ea3c2064ef39e 100644
--- a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs
@@ -39,3 +39,43 @@ impl Scene {
         Ok(())
     }
 }
+
+pub(crate) const EXTENDED_SIZE: usize = 0x400;
+
+/// Encode a native scene header from its retained aliases. The local
+/// statistics and parameter scratch remain part of this same owned record.
+pub(crate) fn encode_extended(output: &mut [u8], owner: super::m3_init_layout::Region,
+    scratch_gpu: super::m3_pool_layout::GpuRegion, scratch_fw: FirmwareVa,
+    scene_list: super::m3_pool_layout::GpuRegion) -> Result<(), Error> {
+    let statistics = owner.at(0x200, 0x40).map_err(|_| Error::Size)?;
+    owner.at(0, EXTENDED_SIZE).map_err(|_| Error::Size)?;
+    if scratch_gpu.size() < 0x4000 || scene_list.size() < 0x4000 { return Err(Error::Size); }
+    let compact = super::agx_render_state::compact(scene_list.base()).map_err(|_| Error::Address)?;
+    let bytes = output.get_mut(..EXTENDED_SIZE).ok_or(Error::Size)?;
+    bytes.fill(0);
+    bytes[..8].copy_from_slice(&scratch_gpu.base().to_le_bytes());
+    bytes[8..16].copy_from_slice(&scratch_fw.get().to_le_bytes());
+    bytes[0x28..0x30].copy_from_slice(&compact.to_le_bytes());
+    bytes[0x40..0x48].copy_from_slice(&statistics.to_le_bytes());
+    Ok(())
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    use super::super::{m3_init_layout::Region, m3_pool_layout::GpuRegion};
+    #[test] fn native_scene_keeps_scratch_aliases_and_local_statistics_distinct() {
+        let owner = Region::new(0xffff_fc25_0010_0000, EXTENDED_SIZE).unwrap();
+        let scratch = GpuRegion::new(0x10_0200_0000, 0x4000).unwrap();
+        let scratch_fw = FirmwareVa::new(0xffff_fc25_0020_0000).unwrap();
+        let list = GpuRegion::new(0x10_0300_0000, 0x4000).unwrap();
+        let mut bytes = [0xa5; EXTENDED_SIZE + 8];
+        encode_extended(&mut bytes, owner, scratch, scratch_fw, list).unwrap();
+        let read = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap());
+        assert_eq!(read(0), scratch.base());
+        assert_eq!(read(8), scratch_fw.get());
+        assert_eq!(read(0x28), 0x300_0000);
+        assert_eq!(read(0x40), owner.at(0x200, 0x40).unwrap());
+        assert_eq!(&bytes[EXTENDED_SIZE..], &[0xa5; 8]);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs
index 9b8d503836d0a..0a2262858556d 100644
--- a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs
@@ -100,3 +100,29 @@ pub(crate) fn fault_buffers()->impl Iterator<Item=(usize,usize)> {
         .chain((PB_FIRST..PB_FIRST+PB_GROUPS).map(|i|(i,256)))
         .chain([(TILEMAP_LABEL,1280),(HEAP_METADATA_LABEL,256)])
 }
+
+pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) -> Result<Allocation,Error> {
+    let mut owner = allocation(index)?;
+    if profile == super::m3_profile::Profile::Legacy { return Ok(owner); }
+    if owner.space == Space::Firmware {
+        owner.address = 0xffff_fc24_0000_0000 + index as u64 * 0x10_0000;
+    }
+    match index {
+        // Extended firmware reads the uncached kernel aliases supplied to
+        // encode_extended; these lower client mappings are GPU-only.
+        PARAMETER_PAGE_LIST | PARAMETER_BLOCK_LIST => owner.access = Access::GpuCachedRw,
+        8 | 20 => owner.size = 0x20000,
+        9 | 21 => { owner.address &= !0x3fff; owner.size = 0x4000; }
+        10 | 22 => {
+            owner.size = 0x4000;
+            owner.gpu_alias = Some(0x72_3000_0000 + u64::from(index == 22) * 0x4000);
+        }
+        GPU_CONTEXT => owner.size = 0x38,
+        NOTIFIER => owner.size = 0x400,
+        EVENT_COUNT => owner.size = 16,
+        BUFFER_MANAGER => owner.size = 0xc0,
+        FRAGMENT_QUEUE | TA_QUEUE => owner.size = 0x24c0,
+        _ => {}
+    }
+    Ok(owner)
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs
index f19717d536f72..7500822c4124c 100644
--- a/drivers/gpu/drm/asahi/m3/m3_soc.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs
@@ -2,48 +2,230 @@
 
 //! The per-SoC facts of the M3 runtime backend.
 //!
-//! Each table describes the configuration of one GPU. This compile-only subset holds only the
-//! facts the M3 boundary carries; the register windows, memory layout, firmware images and
-//! admission of the full table stay with the M3 runtime port. Nothing here is read by a probe.
-
-use super::m3_compute_layout::RegisterSet;
-
-/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor
-/// device-tree values: configuration words where the manager layout keeps 0 or its validated
-/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values
-/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree.
-#[derive(Debug)]
-pub(crate) struct G15RuntimeHwDataB {
-    pub(crate) unk_454: u32,
-    pub(crate) unk_464: u32,
-    pub(crate) unk_a7c: u64,
-    pub(crate) unk_a98: u32,
-    pub(crate) unk_abc: u64,
-    pub(crate) unk_ae4: u32,
-    pub(crate) unk_b20: u32,
-    pub(crate) unk_b24: u64,
-    /// +0xb40, the "UAT enabled" word of the manager layout.
-    pub(crate) unk_554: u32,
-    pub(crate) unk_17b8: u32,
-    /// Unit enable masks, +0x17c0/+0x17c8.
-    pub(crate) unit_mask_a: u64,
-    pub(crate) unit_mask_b: u32,
-    pub(crate) unk_1808: u32,
-    pub(crate) unk_1818: u32,
+//! Each table describes the register windows, memory layout and configuration of one GPU.
+//! Missing configuration is `None`; outstanding runtime requirements are listed in `unported`.
+//! Admission rejects either before reading firmware, mapping GPU registers or starting the
+//! GPU coprocessor ([`Soc::require_complete`]).
+
+use kernel::{c_str, device, prelude::*};
+
+use crate::{
+    m3::hw,
+    m3::initdata::G15RuntimeHwDataB,
+    m3::m3_compute_layout::RegisterSet,
+    m3::m3_board::KnownImage,
+    m3::m3_firmware::Layout,
+    m3::m3_init_storage::IoMap, //
+};
+
+/// The SGX identification words the runtime admits, each compared under its mask.
+pub(crate) struct IdWords {
+    /// SGX+0xd04000: family [31:24], variant [23:16], revision [15:8].
+    pub(crate) version: u32,
+    pub(crate) version_mask: u32,
+    /// SGX+0xd04010: dies [19:16], clusters per die [15:8].
+    pub(crate) counts: u32,
+    pub(crate) counts_mask: u32,
 }
 
+/// The register windows the runtime admits, as CPU physical addresses (device-tree `reg` values
+/// translated through `arm-io` `ranges`).
+pub(crate) struct Windows {
+    /// Base of the GPU coprocessor window (`asc`, ADT `gfx-asc` reg[0]).
+    pub(crate) asc: u64,
+    /// Base of the GPU window (`sgx`, ADT `sgx` reg[0]).
+    pub(crate) sgx: u64,
+    /// Base of the GPU coprocessor mailbox, a 16 KiB window inside `asc`.
+    pub(crate) mailbox: u64,
+}
+
+/// The performance-state table the runtime accepts: checked on the generated InitData before
+/// any GPU register is touched, and again on the uploaded HwDataB.
+#[derive(Copy, Clone, Debug)]
+pub(crate) enum PstateTable {
+    /// A fixed table: `states` states above the off state, the highest at `top_mhz` MHz.
+    Fixed { states: u32, top_mhz: u32 },
+    /// The table of the device tree's operating points: one state per distinct voltage above
+    /// the off state, the highest at the highest frequency
+    /// (`t8122_admission::opp_table_shape`).
+    DeviceTree,
+}
+
+/// Optional userspace features advertised for a SoC (`DRM_ASAHI_GET_PARAMS`). The runtime
+/// implements them the same way on every SoC; a SoC advertises one once it has been validated
+/// there. Soft faults are never advertised: the runtime does not apply `asahi.fault_control`.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct Features {
+    /// `DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY`.
+    pub(crate) compute_wide_visibility: bool,
+    /// `DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY`, while `asahi.m3_early_tiling` is set.
+    pub(crate) fragment_dependency: bool,
+}
+
+/// The MTR temperature-sensor masks of the runtime's HwDataA, where the SoC table sets them
+/// instead of the hardware configuration.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct MtrMasks {
+    /// The fast-die sensor mask, in both HwDataA copies (+0x8ac and +0x1288).
+    pub(crate) fast_die: u64,
+    /// The mask the firmware matches an MTR alarm against (HwDataA +0x1a98). An alarm from a
+    /// sensor outside it, or any alarm while it is 0, is fatal to the firmware.
+    pub(crate) alarm: u64,
+}
+
+/// Where the boot loader exports a SoC's decoded GPU leakage fuse values, in `/chosen`, and the
+/// boot loader switch that makes them this boot's leakage.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct LeakFuse {
+    /// Two cells, value 1 then value 2, encoded as `apple,core-leak-coef` is.
+    pub(crate) values: &'static CStr,
+    /// The switch: the boot loader uses the values exactly when it holds the string "1" (the
+    /// two bytes `1\0`), the form it copies from its configuration line.
+    pub(crate) switch: &'static CStr,
+}
+
+/// Where the runtime places the HwData object (`m3_init_storage::HARDWARE_DATA`), when it is not
+/// the fixed allocation's address, and how the firmware maps it.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct HwDataObject {
+    /// Firmware VA of the object.
+    pub(crate) address: u64,
+    /// Map the object cacheable for the firmware (its atomic updates need cacheable memory)
+    /// instead of uncached.
+    pub(crate) cached: bool,
+}
+
+/// One firmware IO mapping of the runtime's InitData: HwDataB slot, physical address, total
+/// size, element size, writable.
+pub(crate) type IoMapping = (usize, u64, u32, u32, bool);
+
+/// The M3 runtime's facts for one SoC.
 pub(crate) struct Soc {
     /// SoC name, for logs.
     pub(crate) name: &'static str,
+    /// GPU name, for logs.
+    pub(crate) gpu_name: &'static str,
+    pub(crate) chip_id: u32,
+    pub(crate) gpu_variant: hw::GpuVariant,
+    pub(crate) gpu_revision: hw::GpuRevision,
+    pub(crate) gpu_revision_id: hw::GpuRevisionID,
+    /// Root node compatible of the SoC.
+    pub(crate) board: &'static str,
+    /// GPU node compatible.
+    pub(crate) gpu: &'static str,
+    /// Boards the runtime starts on by default and registers a render node on. On a T8122 it
+    /// starts on no other board, whatever `asahi.m3_backend` says (`m3_params::t8122_backend`).
+    pub(crate) validated_boards: &'static [&'static [u8]],
+    /// GPU clusters, and core slots per cluster.
+    pub(crate) clusters: u32,
+    pub(crate) cores_per_cluster: u32,
+    /// Accepted compatible lists of the GPU coprocessor mailbox.
+    pub(crate) mailbox_compatibles: &'static [&'static [u8]],
+    /// The mailbox interrupts: send-empty, send-not-empty, recv-empty, recv-not-empty, as the
+    /// mailbox node's `interrupts` cells (three per interrupt, or four with the die cell of a
+    /// multi-die interrupt controller).
+    pub(crate) mailbox_interrupts: &'static [u32],
+    /// The register windows of the GPU node and its mailbox.
+    pub(crate) windows: Windows,
+    /// The identification words the runtime admits.
+    pub(crate) id: IdWords,
+    /// GPU firmware images the runtime can identify.
+    pub(crate) images: &'static [KnownImage],
+    /// The layout of the loaded firmware the runtime accepts.
+    pub(crate) firmware: Option<&'static Layout>,
+    /// The hardware configuration of the InitData.
+    pub(crate) hwcfg: Option<&'static hw::HwConfig>,
+    /// The firmware IO mappings of the InitData.
+    pub(crate) io_mappings: Option<&'static [IoMapping]>,
+    /// The runtime's mapping of each of those IO mappings: the CPU physical block and the
+    /// firmware VA it is mapped at. One per entry of `io_mappings`, covering it.
+    pub(crate) iomaps: Option<&'static [IoMap]>,
+    /// The performance-state table the runtime accepts.
+    pub(crate) pstates: PstateTable,
+    /// The SGX write (offset, value) made after the identity checks, before the firmware starts.
+    pub(crate) sgx_setup: Option<(usize, u32)>,
     /// The runtime InitData's HwDataB configuration words and unit masks.
     pub(crate) hwdata_b: Option<&'static G15RuntimeHwDataB>,
+    /// The optional userspace features advertised.
+    pub(crate) features: Features,
+    /// Whether the power-management coefficients, the leakage coefficients and the operating
+    /// points are this machine's, added by the boot loader from its ADT, rather than static
+    /// device-tree values. Admission then requires every one of them (see
+    /// `m3_board::check_boot_loader_power`), with no driver default standing in except the
+    /// idle-off standby timer, which an ADT may lack.
+    pub(crate) power_from_boot_loader: bool,
+    /// Code the runtime runs after admission that still holds another SoC's values.
+    pub(crate) unported: &'static [&'static str],
+    /// The GPU registers read after every job by default (`asahi.m3_retire_mmio` bits).
+    pub(crate) retire_mmio: u64,
+    /// The MTR sensor masks, when they are not the hardware configuration's.
+    pub(crate) mtr_masks: Option<MtrMasks>,
+    /// The HwData object's placement and mapping, when they are not the fixed allocation's.
+    pub(crate) hwdata_object: Option<HwDataObject>,
     /// The register lists the GPU's firmware expects in compute and render commands.
     pub(crate) registers: RegisterSet,
+    /// Globals +0x7d0, which gates the firmware's frequency-feedback cap: the firmware measures
+    /// the GPU clock against the requested performance state and, while this is 1, caps the
+    /// state on a shortfall (`asahi.m3_ut_engagement` overrides it).
+    pub(crate) ut_engagement: u32,
+    /// The highest GPU power target, in mW, while the boot loader's power model is a stand-in:
+    /// the operating points' powers are scaled down together so that the highest is this.
+    pub(crate) power_target_cap_mw: Option<u32>,
     /// Words of the generated HwData object (offset into the object, value) that this SoC's
     /// firmware takes with other values than the shared builder writes.
     pub(crate) hwdata_words: &'static [(usize, u32)],
     /// The same for the Globals object.
     pub(crate) globals_words: &'static [(usize, u32)],
+    /// The boot loader's decoded GPU leakage fuse, when it exports one for this SoC.
+    pub(crate) leak_fuse: Option<LeakFuse>,
+    /// Whether each operating point may give each cluster its own voltage (the per-cluster,
+    /// binned voltage tables of T6031). Otherwise every cluster of a point must have the same.
+    pub(crate) per_cluster_voltages: bool,
+}
+
+impl Soc {
+    /// Reject incomplete configuration before accessing the GPU.
+    pub(crate) fn require_complete(&self, dev: &device::Device) -> Result {
+        let mut missing = 0u32;
+        let mut note = |what: &str| {
+            dev_info!(dev, "M3 {}: missing {}\n", self.gpu_name, what);
+            missing += 1;
+        };
+        if !self.images.iter().any(|image| image.initdata_magic.is_some()) {
+            note("the identity of the loaded GPU firmware image and its InitData version");
+        }
+        if self.firmware.is_none() {
+            note("the layout of the loaded GPU firmware (segment sizes and VAs)");
+        }
+        if self.hwcfg.is_none() {
+            note("the hardware configuration of the InitData (HwConfig)");
+        }
+        if self.io_mappings.is_none() {
+            note("the firmware IO mappings of the InitData");
+        }
+        if self.iomaps.is_none() {
+            note("the runtime's mapping of the firmware IO mappings (IO maps)");
+        }
+        if self.sgx_setup.is_none() {
+            note("the SGX setup write made before the firmware starts");
+        }
+        if self.hwdata_b.is_none() {
+            note("the runtime HwDataB configuration words and unit masks");
+        }
+        for &what in self.unported {
+            note(what);
+        }
+        if missing == 0 {
+            return Ok(());
+        }
+        dev_err!(
+            dev,
+            "M3 {}: {} configuration requirements missing; GPU startup disabled\n",
+            self.gpu_name,
+            missing
+        );
+        Err(ENODEV)
+    }
 }
 
 /// The HwDataB configuration words and unit masks of the T6030 runtime InitData, as checked on
@@ -77,24 +259,115 @@ pub(crate) static T8122_HWDATA_B: G15RuntimeHwDataB = G15RuntimeHwDataB {
 };
 
 /// Every SoC the M3 runtime has a table for.
-pub(crate) static SOCS: [&Soc; 2] = [&T6030, &T8122];
-
-/// T6030 (M3 Pro, G15S): one die, two clusters of ten core slots.
-pub(crate) static T6030: Soc = Soc {
-    name: "T6030",
-    hwdata_b: Some(&T6030_HWDATA_B),
-    registers: RegisterSet::G15S,
-    hwdata_words: &[],
-    globals_words: &[],
-};
+pub(crate) static SOCS: [&Soc; 1] = [&T8122];
+
+/// The table of the SoC with chip id `chip_id`, if the M3 runtime has one.
+pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> {
+    SOCS.iter().copied().find(|soc| soc.chip_id == chip_id)
+}
 
 /// T8122 (M3, G15G): one die, one cluster of ten core slots (eight or ten of them active).
+///
+/// The compatibles, the mailbox and its interrupts are the T8122 device tree's. The identity is
+/// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20,
+/// core slots in the first core-mask word only), with the die count of the AGX3 identification
+/// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration
+/// comes from the boot loader. Startup uses the guarded `t8122_start` path, which selects this
+/// table's values by default on admitted Air boards and adds a performance-state cap. Explicit
+/// `asahi.t8122_start=0` disables startup; firmware, board and resource admission still apply.
 pub(crate) static T8122: Soc = Soc {
     name: "T8122",
+    gpu_name: "G15G",
+    chip_id: 0x8122,
+    gpu_variant: hw::GpuVariant::G,
+    gpu_revision: hw::GpuRevision::C0,
+    gpu_revision_id: hw::GpuRevisionID::C0,
+    board: "apple,t8122",
+    gpu: "apple,agx-t8122",
+    // The M3 MacBook Airs (13" J613, 15" J615). Allowed, not validated: the runtime has not run
+    // on either, and starts only behind a boot loader that hands the GPU over.
+    validated_boards: &[b"apple,j613", b"apple,j615"],
+    clusters: 1,
+    cores_per_cluster: 10,
+    mailbox_compatibles: &[
+        b"apple,t8122-agx-asc-mailbox\0",
+        b"apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0",
+        b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0",
+    ],
+    mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4],
+    // The J613 ADT: sgx reg[0] (child 0x80000000 + arm-io 0x210000000) and gfx-asc reg[0]; the
+    // mailbox at asc + 0x8000 (t8122-gpu.dtsi). The same addresses as on T6030.
+    windows: Windows {
+        asc: 0x2_9240_0000,
+        sgx: 0x2_9000_0000,
+        mailbox: 0x2_9240_8000,
+    },
+    id: IdWords {
+        version: 0x0702_2000,
+        version_mask: 0xffff_ff00,
+        // One die, one cluster per die; the other fields are not known.
+        counts: 0x0001_0100,
+        counts_mask: 0x000f_ff00,
+    },
+    images: &crate::m3::m3_board::KNOWN_IMAGES_T8122,
+    firmware: Some(&crate::m3::m3_firmware::T8122_LAYOUT),
+    hwcfg: Some(&hw::t8122::HWCONFIG_T8122),
+    io_mappings: Some(&crate::m3::m3_adt_config::T8122_IO_MAPPINGS),
+    iomaps: Some(&crate::m3::m3_adt_config::T8122_IOMAPS),
+    // The boot loader's ladder from this machine's ADT (J613: eight voltages, up to 1338 MHz).
+    pstates: PstateTable::DeviceTree,
+    // The same SGX setup write as on T6030.
+    sgx_setup: Some((0xd14000, 0x70001)),
     hwdata_b: Some(&T8122_HWDATA_B),
+    // Neither is validated on G15G yet; userspace keeps its default ordering and visibility.
+    features: Features {
+        compute_wide_visibility: false,
+        fragment_dependency: false,
+    },
+    power_from_boot_loader: true,
+    unported: &[
+        "M3 Air startup requires the guarded t8122_start path and its performance-state cap",
+    ],
+    // The performance state only. On the single-cluster G15G, reading the engine-busy or the
+    // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s;
+    // the performance-state read alone does not, and a per-frame readback saw no job retired
+    // before its writes were visible.
+    retire_mmio: 4,
+    // The fast-die controller's sensors (0x4248, both copies), and every sensor the MTR block
+    // enables as the alarm mask (sensors 3, 6, 8, 9, 11 and 14: 0x4b48, the upper half of the
+    // block's configuration word 0x4b480003). With the second fast-die copy at 0, or an alarm
+    // mask without sensor 8, the firmware stopped on its first MTR alarm.
+    mtr_masks: Some(MtrMasks {
+        fast_die: 0x4248,
+        alarm: 0x4b48,
+    }),
+    // The firmware's MTR alarm handler does a 64-bit atomic update of HwDataA +0x4350. The fixed
+    // allocation ends the 0x8a04-byte object at its page end, leaving HwDataA (+0x4580) only
+    // 4-byte aligned, and the update took an alignment fault. Starting the object 0x8a80 bytes
+    // before the same page end puts HwDataA on a 16-byte boundary. Aligned, the update then
+    // faulted as an unsupported atomic on the uncached mapping: map it cacheable.
+    hwdata_object: Some(HwDataObject {
+        address: T8122_HWDATA_ADDRESS,
+        cached: true,
+    }),
     registers: RegisterSet::G15G,
+    // On a T8122 with the 14.8.3 system firmware the shader clock runs at 3/4 of every requested
+    // state, so with the cap engaged the GPU stays at state 2 (462 MHz effective) under any load.
+    // With 0 it reaches the requested states (state 8: about 1000 MHz effective).
+    ut_engagement: 0,
+    // The boot loader's T8122 power model is a stand-in that overstates the power (its highest
+    // operating point is about 30 W). Until each Mac's fused leakage gives a real one, scale it
+    // to a 22 W target, about the GPU power budget of an M3 MacBook Air.
+    power_target_cap_mw: Some(22_000),
     hwdata_words: &T8122_HWDATA_WORDS,
     globals_words: &T8122_GLOBALS_WORDS,
+    // Exported on a J613 on every boot; used only with the boot loader's switch, which also makes
+    // value 1 its apple,core-leak-coef and the base of its operating points' powers.
+    leak_fuse: Some(LeakFuse {
+        values: c_str!("asahi,t8122-gpu-leak-fuse"),
+        switch: c_str!("asahi,t8122-gpu-fuse-leakage"),
+    }),
+    per_cluster_voltages: false,
 };
 
 /// The T8122 HwData words that differ from the shared builder's (offsets into the object;
@@ -104,7 +377,7 @@ pub(crate) static T8122: Soc = Soc {
 ///   +0x1804, +0x1814 and +0x1860 and all-ones at +0x1848);
 /// - HwDataA +0x11e0: 30 (T6030: 40), a word of the shader-engine controller block;
 /// - HwDataA +0x1290: 125, and HwDataA +0x424c: 24000000, the 24 MHz reference clock.
-static T8122_HWDATA_WORDS: [(usize, u32); 17] = [
+pub(crate) static T8122_HWDATA_WORDS: [(usize, u32); 17] = [
     (0xa30, 0),
     (0xa34, 4),
     (0x17e0, 1),
@@ -126,4 +399,18 @@ static T8122_HWDATA_WORDS: [(usize, u32); 17] = [
 
 /// The T8122 Globals words that differ from the shared builder's: +0x9bc, the CDM backoff
 /// timeout (4) with the three bytes after it 0 (T6030: 1, 0, 0).
-static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)];
+pub(crate) static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)];
+
+/// The T8122 HwData object's firmware VA: 0x8a80 bytes before the end of the fixed allocation's
+/// last page.
+pub(crate) const T8122_HWDATA_ADDRESS: u64 = 0xffff_fc20_4070_4000 - 0x8a80;
+
+// The fixed HwData allocation (`m3_init_storage::allocation`): 0x8a04 bytes ending at the page
+// end 0xfffffc2040704000. The moved object stays inside the same pages, and HwDataA is 16-byte
+// aligned.
+const _: () = {
+    let (fixed, size) = (0xffff_fc20_406f_b5fc_u64, 0x8a04_u64);
+    assert!(T8122_HWDATA_ADDRESS & !0x3fff == fixed & !0x3fff);
+    assert!(T8122_HWDATA_ADDRESS + size <= (fixed + size + 0x3fff) & !0x3fff);
+    assert!((T8122_HWDATA_ADDRESS + crate::m3::m3_adt_config::HWDATA_A as u64) % 16 == 0);
+};
diff --git a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs
index 199b7082b26c2..6eeafea666e74 100644
--- a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs
@@ -66,3 +66,55 @@ pub(crate) fn manager_misc(out: &mut [u8]) -> Result<(), Error> {
     word(bytes, 0x20, 1); // cpu_flag / reset
     Ok(())
 }
+
+pub(crate) const EXTENDED_CONTEXT_SIZE: usize = 0x38;
+pub(crate) const EXTENDED_NOTIFIER_SIZE: usize = 0x200;
+
+pub(crate) fn context_extended(output: &mut [u8]) -> Result<(), Error> {
+    let bytes = output.get_mut(..EXTENDED_CONTEXT_SIZE).ok_or(Error::Size)?;
+    bytes.fill(0);
+    bytes[..2].fill(0xff);
+    bytes[5] = 1;
+    bytes[0x26] = 2;
+    bytes[0x33] = 0xff;
+    Ok(())
+}
+
+impl Notifier {
+    pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> {
+        if self.context >= 64 { return Err(Error::Address); }
+        let bytes = output.get_mut(..EXTENDED_NOTIFIER_SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        bytes[..8].copy_from_slice(&self.threshold.get().to_le_bytes());
+        word(bytes, 0x10, 0x50);
+        word(bytes, NOTIFIER_CONTEXT, self.context);
+        bytes[0xa8..0xb0].fill(0xff);
+        word(bytes, 0x114, 1);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    #[test] fn registration_state_is_left_to_firmware() {
+        let mut bytes = [0xa5; EXTENDED_CONTEXT_SIZE + 8];
+        context_extended(&mut bytes).unwrap();
+        assert_eq!(&bytes[..6], &[255, 255, 0, 0, 0, 1]);
+        assert_eq!(bytes[0x26], 2);
+        assert_eq!(bytes[0x33], 255);
+        assert_eq!(&bytes[EXTENDED_CONTEXT_SIZE..], &[0xa5; 8]);
+    }
+    #[test] fn notifier_has_zero_generation_and_distinct_firmware_storage() {
+        let notifier = Notifier { threshold: FirmwareVa::new(0xffff_fc21_7000_0000).unwrap(), context: 3 };
+        let mut bytes = [0xa5; EXTENDED_NOTIFIER_SIZE + 8];
+        notifier.encode_extended(&mut bytes).unwrap();
+        assert_eq!(&bytes[8..12], &[0; 4]);
+        assert_eq!(&bytes[0xa8..0xb0], &[255; 8]);
+        assert_eq!(&bytes[0x114..0x118], &[1, 0, 0, 0]);
+        assert_eq!(&bytes[EXTENDED_NOTIFIER_SIZE..], &[0xa5; 8]);
+        let mut short = [0xa5; NOTIFIER_SIZE];
+        assert!(notifier.encode_extended(&mut short).is_err());
+        assert_eq!(short, [0xa5; NOTIFIER_SIZE]);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs
new file mode 100644
index 0000000000000..155bda714e9d9
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs.
+use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray};
+use crate::{file,m3::agx_uapi,mmu,queue,m3::m3_drm::Shared};
+use core::sync::atomic::{AtomicBool, Ordering};
+#[derive(Default)]
+pub(crate) struct Completion;
+#[vtable]
+impl FenceOps for Completion {
+    fn get_driver_name<'a>(self:&'a FenceObject<Self>)->&'a CStr {c_str!("asahi")}
+    fn get_timeline_name<'a>(self:&'a FenceObject<Self>)->&'a CStr {c_str!("m3-native")}
+}
+pub(crate) struct Destination { mapping: Arc<mmu::KernelMapping>, offset: usize }
+impl Destination {
+    fn resolve(objects: Pin<&xarray::XArray<KBox<file::Object>>>,
+        timestamp: agx_uapi::UapiTimestamp) -> Result<Option<Self>> {
+        if timestamp.handle == 0 { return Ok(None); }
+        let guard = objects.lock();
+        let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?;
+        match &*object {
+            file::Object::TimestampBuffer(buffer) => {
+                let mapping = buffer.firmware_mapping()?;
+                let offset = timestamp.offset as usize;
+                if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() {
+                    return Err(EINVAL);
+                }
+                Ok(Some(Self { mapping, offset }))
+            }
+        }
+    }
+    fn firmware_address(&self) -> u64 { self.mapping.iova() + self.offset as u64 }
+}
+
+#[derive(Copy,Clone)]
+pub(crate) enum Command {
+    Compute(crate::m3::agx_compute::Control),
+    Render{command:agx_uapi::UapiRenderCommand,usc:u64},
+}
+pub(crate) struct Packet {
+    pub(crate) vm:mmu::Vm,
+    pub(crate) commands:KVec<Command>,
+    timestamps:KVec<[Option<Destination>;4]>,
+    pub(crate) attachments:KVec<crate::m3::agx_attachments::Attachments>,
+    pub(crate) wide_visibility:KVec<bool>,
+    pub(crate) render_attachments:KVec<[crate::m3::agx_attachments::Attachments;2]>,
+    completion:UserFence<Completion>,
+    finish_claimed: AtomicBool,
+    /// Set before a batch with any of this packet's commands can change firmware-visible state,
+    /// and never cleared.
+    published: AtomicBool,
+    /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first.
+    pub(crate) urgency:u8,
+    /// Who submitted the packet, for the failure log only.
+    pub(crate) owner:Owner,
+    vm_job:Pin<KBox<Mutex<Option<mmu::VmJobGuard>>>>,
+}
+/// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged:
+/// a renamed task leaves a stale name, which is harmless.
+#[derive(Copy,Clone)]
+pub(crate) struct Owner {pub(crate) pid:i32,process:[u8;16],thread:[u8;16]}
+impl Owner {
+    fn current()->Self {
+        let task=kernel::current!();
+        let leader=task.group_leader();
+        Self {pid:leader.pid(),process:crate::m3::m3_client::task_comm(leader),
+            thread:crate::m3::m3_client::task_comm(task)}
+    }
+    fn name(comm:&[u8;16])->&str {
+        let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len());
+        core::str::from_utf8(&comm[..len]).unwrap_or("?")
+    }
+    pub(crate) fn process(&self)->&str {Self::name(&self.process)}
+    pub(crate) fn thread(&self)->&str {Self::name(&self.thread)}
+}
+impl Packet {
+    pub(crate) fn profile_id(&self)->usize {self as *const Self as usize}
+    pub(crate) fn timestamp_addresses(&self,index:usize)->[u64;2] {
+        core::array::from_fn(|i|self.timestamps[index][i].as_ref().map_or(0,Destination::firmware_address))
+    }
+    pub(crate) fn render_timestamp_addresses(&self,index:usize)->[[u64;2];2] {
+        core::array::from_fn(|stage| core::array::from_fn(|i|
+            self.timestamps[index][stage*2+i].as_ref().map_or(0,Destination::firmware_address)))
+    }
+    /// Called only after all published commands retired or before any publication.
+    pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());}
+    /// Called before the first firmware-visible change of a batch that includes this packet.
+    pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);}
+    /// Whether the firmware may have seen any of this packet's commands.
+    pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)}
+    pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));}
+    pub(crate) fn finish(&self,result:Result) {
+        // Retirement, cancellation and a queued scheduler timeout can race.
+        // Claim error/signal ownership once; this flag is NOT proof that the
+        // fence is signaled or that hardware DMA has stopped.
+        if self.finish_claimed.swap(true, Ordering::Relaxed) { return; }
+        if let Err(e)=result {
+            // A guard still held here belongs to work the firmware may still run: block
+            // mapping changes before the error fence is visible (as G15 does), so userspace
+            // can't unmap memory under it. It stays with the packet until the runtime drops
+            // the packet, after the firmware has provably stopped.
+            if let Some(guard)=self.vm_job.lock().as_ref() {guard.quarantine();}
+            if let Some(status) = self.vm.status() { status.record(e); }
+            self.completion.set_error(e);
+        }
+        // A scheduler job/fence and the runtime's last packet can outlive
+        // hardware retirement. They must not keep the VM's active-job count
+        // nonzero: GEM closes otherwise accumulate across completed frames.
+        // Signalling an error is not retirement.
+        crate::m3::agx_memory::publish();
+        let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming);
+        let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0};
+        self.completion.signal();
+        if profile {
+            let after=crate::m3::m3_runtime::physical_counter();
+            let status=match result {Ok(())=>0,Err(e)=>e.to_errno()};
+            pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n",
+                self.vm.id(),self.profile_id(),self.commands.len(),before,after,status);
+        }
+    }
+}
+pub(crate) struct Job {shared:Shared,packet:Arc<Packet>}
+/// Timeout state of a [`Job`], owned by the scheduler's timeout worker.
+pub(crate) struct JobTimeout {shared:Shared,packet:Arc<Packet>}
+impl sched::JobImpl for Job {
+    type TimeoutData = JobTimeout;
+    fn timeout_data(&self)->JobTimeout {
+        JobTimeout {shared:self.shared.clone(),packet:self.packet.clone()}
+    }
+    fn run(job:&mut Self)->Result<Option<Fence>> {
+        // Queue the packet and return its fence; the completion worker publishes and retires
+        // it, so the scheduler can hand over further ready jobs meanwhile.
+        if let Err(e)=crate::m3::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) {
+            pr_err!("M3 scheduler: submission failed {:?}\n",e);
+            // submit() failed before this packet could be published.
+            job.packet.release_guard();job.packet.finish(Err(e));
+        }
+        Ok(Some(Fence::from_fence(&job.packet.completion)))
+    }
+    fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status {
+        // Retain the publishing branch's explicit diagnostic override.
+        if !crate::m3::m3_params::timeout_nohang() {
+            if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();}
+            crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet);
+            job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice;
+        }
+        let guard=job.shared.lock();
+        let Some(runtime)=guard.as_ref() else {
+            job.packet.finish(Err(ENODEV));
+            return sched::Status::NoDevice;
+        };
+        // The scheduler's deadline covers the whole packet, including time queued behind
+        // other batches. The completion worker owns each batch's bounded hardware-completion
+        // checks. Reinsert a healthy packet instead of turning a late timeout into a global
+        // device loss or corrupting an already signaled fence.
+        if runtime.health().healthy() { return sched::Status::NoHang; }
+        drop(guard);
+        crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet);
+        job.packet.finish(Err(EIO));
+        sched::Status::NoDevice
+    }
+    fn cancel(job:&mut Self) {
+        crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet);
+        job.packet.finish(Err(ECANCELED));
+    }
+}
+struct AddressSpace<'a>(&'a mmu::Vm);
+impl agx_uapi::GpuAddressSpace for AddressSpace<'_> {
+    fn covers(&self,address:u64,size:u64,access:agx_uapi::GpuAccess)->bool {
+        let Some(end)=address.checked_add(size) else {return false;};
+        let (r,w)=match access {agx_uapi::GpuAccess::Read=>(true,false),
+            agx_uapi::GpuAccess::Write=>(false,true),agx_uapi::GpuAccess::ReadWrite=>(true,true)};
+        address>=0x4000 && end<=(1u64<<42)-0x8000
+            && !self.0.driver_range_overlaps(address..end) && self.0.covers_range(address,size,r,w)
+    }
+}
+pub(crate) struct Queue {
+    entity:sched::Entity<Job>,_scheduler:Arc<sched::Scheduler<Job>>,
+    shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts,
+    /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`).
+    early_tiling:bool,
+    wide_visibility:bool,
+    urgency:u8,
+}
+impl Queue {
+    pub(crate) fn new(shared:Shared,scheduler:Arc<sched::Scheduler<Job>>,vm:mmu::Vm,priority:u32,usc:u64,
+        early_tiling:bool, wide_visibility:bool)->Result<Self> {
+        agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?;
+        // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME.
+        let urgency=3u8.saturating_sub(priority.min(3) as u8);
+        let priority=match priority {3=>sched::Priority::Low,2=>sched::Priority::Normal,
+            1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)};
+        Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler,
+            shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?,
+            early_tiling,wide_visibility,urgency})
+    }
+}
+impl queue::Queue for Queue {
+    fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec<file::SyncItem>,in_sync_count:usize,
+        raw:&[u8],objects:Pin<&xarray::XArray<KBox<file::Object>>>)->Result {
+        let vm_job=self.vm.retain_job()?;
+        let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new();let mut render_attachments=KVec::new();
+        while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? {
+            if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command {
+                if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) {
+                    pr_err!("M3 render rejected {:?}: flags={:#x} size={}x{} layers={} samples={} utile={}x{} sample_size={} depth={:#x} stencil={:#x} zls={:#x} samplers={} heap={:#x}\n",e,payload.flags,payload.width,payload.height,payload.layers,payload.samples,payload.utile_width,payload.utile_height,payload.sample_size,payload.depth.base,payload.stencil.base,payload.zls_control,payload.sampler_count,payload.sampler_heap);
+                    return Err(e);
+                }
+                agx_uapi::validate_attachments(&AddressSpace(&self.vm),&vertex_attachments).map_err(|_|EINVAL)?;
+                agx_uapi::validate_attachments(&AddressSpace(&self.vm),&fragment_attachments).map_err(|_|EINVAL)?;
+                commands.push(Command::Render{command:payload,usc:self.usc},GFP_KERNEL)?;
+                timestamps.push([Destination::resolve(objects,payload.vertex_timestamps.start)?,
+                    Destination::resolve(objects,payload.vertex_timestamps.end)?,
+                    Destination::resolve(objects,payload.fragment_timestamps.start)?,
+                    Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?;
+                flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?;
+                render_attachments.push([encode_attachments(&self.vm,&vertex_attachments)?,encode_attachments(&self.vm,&fragment_attachments)?],GFP_KERNEL)?;
+                visibility.push(false,GFP_KERNEL)?;
+                continue;
+            }
+            let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);};
+            // Other engines and helpers need their own M3 firmware translation.
+            let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32;
+            if payload.flags & !wide !=0 || (!self.wide_visibility && payload.flags & wide != 0)
+                || payload.sampler_count!=0 || payload.sampler_heap!=0 {
+                pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle);
+                return Err(ENOTSUPP);
+            }
+            visibility.push(payload.flags & wide !=0,GFP_KERNEL)?;
+            // The common translator validates the original command/attachment
+            // addresses; this M3 scheduling hint changes none of them.
+            payload.flags &= !wide;
+            render_attachments.push([crate::m3::agx_attachments::Attachments::EMPTY;2],GFP_KERNEL)?;
+            flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?;
+            let c=agx_uapi::translate_compute_command(payload,attachments,
+                agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000},
+                &AddressSpace(&self.vm)).map_err(|e| {
+                    pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper);
+                    EINVAL
+                })?;
+            crate::m3::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?;
+            timestamps.push([Destination::resolve(objects,payload.timestamps.start)?,
+                Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?;
+            commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?;
+        }
+        parser.finish().map_err(|_|EINVAL)?;
+        let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility,render_attachments,
+            completion:self.fences.new_fence(0,Completion)?.into(),
+            finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(),
+            vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?;
+        let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?;
+        for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}}
+        let mut job=job.arm();let finished=job.fences().finished();job.push();
+        for mut sync in syncs {if let Some(chain)=sync.chain_fence.take() {
+            sync.syncobj.add_point(chain,&finished,sync.timeline_value);
+        } else {sync.syncobj.replace_fence(Some(&finished));}}
+        Ok(())
+    }
+}
+
+fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList)
+    -> Result<crate::m3::agx_attachments::Attachments> {
+    use crate::m3::agx_attachments::{Attachments, CAPACITY};
+    use agx_uapi::{GpuAddressSpace, GpuAccess};
+    if list.as_slice().len()>CAPACITY {return Err(E2BIG);}
+    let mut ranges = [(0, 0); CAPACITY];
+    for (range, entry) in ranges.iter_mut().zip(list.as_slice()) {
+        *range = Attachments::range(entry.address, entry.size).map_err(|_| EINVAL)?;
+        if !AddressSpace(vm).covers(range.0, range.1, GpuAccess::Write) {
+            return Err(EINVAL);
+        }
+    }
+    let mut out=Attachments::new(&ranges[..list.as_slice().len()]).map_err(|_| EINVAL)?;
+    // G15S RTKit-2419 uses the legacy 0x17 cache attribute; retain the
+    // M4 range normalization and ownership checks, not its 0x18 attribute.
+    for i in 0..list.as_slice().len() {out.0[i*16+12..i*16+14].copy_from_slice(&0x17u16.to_le_bytes());}
+    Ok(out)
+}
+
+fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpace<'_>,
+    early_tiling: bool) -> Result {
+    use agx_uapi::{GpuAddressSpace, GpuAccess};
+    if r.sample_size % 8 != 0 {return Err(ENOTSUPP);}
+    let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 };
+    // Early tiling (render flag bit 5) only where it is advertised.
+    let stage_scope=if early_tiling && crate::m3::m3_params::early_tiling() {1<<5} else {0};
+    if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4)
+        || r.sampler_count != 0 || r.sampler_heap != 0
+        || r.vertex_helper != empty || r.fragment_helper != empty
+        || r.stencil.compression_base != 0 || r.stencil.compression_stride != 0
+        || r.ppp_control & !0x203 != 0 { return Err(ENOTSUPP); }
+    // Native 26A428 descriptors witness the ZLS metadata registers and mirrors
+    // for single-sample, single-layer depth. Keep other layouts unqualified.
+    if (r.depth.compression_base != 0 || r.stencil.compression_base != 0)
+        && (r.samples != 1 || r.layers != 1) {return Err(ENOTSUPP);}
+    // sample_size describes color tilebuffer storage. Depth/stencil-only
+    // passes legitimately require zero bytes of color storage.
+    if r.layers == 0 || r.layers > 2048 || r.width == 0 || r.height == 0 || r.width > 16384 || r.height > 16384
+        || !matches!(r.utile_width,16|32) || !matches!(r.utile_height,16|32)
+        || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 {
+        return Err(EINVAL);
+    }
+    crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?;
+    if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read)
+        || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) {
+        return Err(EINVAL);
+    }
+    for (p,access) in [(r.depth_bias_base,GpuAccess::Read),(r.occlusion_query_base,GpuAccess::Write)] {
+        if p != 0 && (p & 7 != 0 || !space.covers(p,8,access)) { return Err(EINVAL); }
+    }
+    for zls in [r.depth,r.stencil] { agx_uapi::validate_zls(space,zls,r.layers).map_err(|_| EINVAL)?; }
+    let window = agx_uapi::QueueUscWindow { base:usc,user_start:0x4000,user_end:((1u64 << 42) - 2 * mmu::UAT_PGSZ as u64) };
+    for p in [r.background,r.end_of_tile,r.partial_background,r.partial_end_of_tile] {
+        window.full_program(space,p.usc).map_err(|_| EINVAL)?;
+        if !space.covers(usc+u64::from(p.usc & !63),64,GpuAccess::Read) { return Err(EINVAL); }
+    }
+    Ok(())
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs
index 92d533efa8234..2cf5e18914043 100644
--- a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs
@@ -91,3 +91,58 @@ impl Barrier {
         Ok(())
     }
 }
+
+impl InitBufferManager {
+    /// The extended rebuild path requires a zero staged-block count. Appending
+    /// already-staged blocks would leave a previous hardware descriptor live.
+    pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> {
+        if self.context >= 64 || self.slot >= 32 { return Err(Error::Address); }
+        Self { context: self.context, slot: self.slot, block_count: 0,
+            manager: self.manager, stamp: self.stamp }.encode(output)
+    }
+}
+
+impl Barrier {
+    pub(crate) const EXTENDED_SIZE: usize = 0x40;
+
+    pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> {
+        if self.event >= 128 || self.wait_value == 0 || self.self_value == 0 {
+            return Err(Error::Address);
+        }
+        let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?;
+        bytes.fill(0);
+        u32_at(bytes, 0, 4);
+        address_at(bytes, 4, self.stamp);
+        address_at(bytes, 12, self.stamp);
+        u32_at(bytes, 0x14, self.wait_value);
+        u32_at(bytes, 0x20, self.event);
+        u32_at(bytes, 0x24, self.self_value);
+        if matches!(self.dependency, Dependency::PreviousPass) { u32_at(bytes, 0x3c, 1); }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    #[test] fn full_rebuild_never_uses_the_legacy_staged_count() {
+        let mut bytes = [0xa5; INIT_BM_SIZE];
+        InitBufferManager { context: 3, slot: 7, block_count: 2048,
+            manager: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), stamp: 0x100 }
+            .encode_extended(&mut bytes).unwrap();
+        assert_eq!(&bytes[12..20], &[0; 8]);
+    }
+    #[test] fn extended_dependencies_use_their_native_tail_and_preserve_trailing_storage() {
+        for previous in [false, true] {
+            let mut bytes = [0xa5; Barrier::EXTENDED_SIZE + 8];
+            Barrier { stamp: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), wait_value: 0x100,
+                event: 2, self_value: 0x200, uuid: 9,
+                dependency: if previous { Dependency::PreviousPass } else { Dependency::TilerToFragment } }
+                .encode_extended(&mut bytes).unwrap();
+            assert_eq!(&bytes[0x18..0x20], &[0; 8]);
+            assert_eq!(&bytes[0x20..0x24], &[2, 0, 0, 0]);
+            assert_eq!(u32::from_le_bytes(bytes[0x3c..0x40].try_into().unwrap()), previous.into());
+            assert_eq!(&bytes[Barrier::EXTENDED_SIZE..], &[0xa5; 8]);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs
new file mode 100644
index 0000000000000..e30f8fe011332
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs
@@ -0,0 +1,504 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Thermal limit of the M3 runtime backend's GPU performance cap (`asahi.m3_thermal`).
+//!
+//! Without it (`off`, the default with the built-in InitData image) the firmware is given a
+//! fixed highest performance state (5 of 8 with device-tree InitData: 1056 of 1380 MHz on
+//! J516S) and nothing watches temperatures. The firmware's own die temperature controller is
+//! off in the InitData this driver builds. With device-tree InitData the default is `on`;
+//! `asahi.m3_thermal=off` restores the fixed cap exactly.
+//!
+//! With it, the InitData gives the firmware the whole table as its highest state (the
+//! "ceiling"), and what bounds the firmware is the runtime cap: the Globals performance-state
+//! cap and power-interface state targets, which the firmware keeps reading while it runs (a
+//! host moves the cap with a single store, with no message). The runtime cap starts at the
+//! fixed cap of `off` (the "safe cap") and never goes below it, so at worst the GPU runs as it
+//! does without the limit.
+//!
+//! - `hold`: the runtime cap stays at the safe cap for the whole boot. The GPU then runs exactly
+//!   as with `off`, provided the firmware follows the runtime cap; the per-job check below
+//!   proves or disproves that before `on` is used.
+//! - `on`: the runtime cap follows the hottest SoC die temperature, read by a work item every
+//!   500 ms while the GPU is busy (every 2 s when idle) from the thermal zones in [`ZONES`]
+//!   that exist: the SMC's die sensors and the PMP's CPU cluster sensors (see [`ZONES`]):
+//!   * at or above `critical` (`hot` + 8 C): the safe cap at once;
+//!   * at or above `hot` (`asahi.m3_thermal_hot`, default 80 C): one state lower, at most every
+//!     500 ms and once per reading;
+//!   * at or below `cool` (`hot` - 10 C): one state higher, at most every 2 s and once per
+//!     reading, up to the ceiling;
+//!   * no zone, a zone that cannot be read, a reading outside 10..=125 C, one older than 3 s
+//!     or one that has not changed at all for 60 s: the safe cap at once.
+//!
+//! The defaults leave a wide margin below the limits the boot firmware's device tree gives this
+//! machine (a 94 C GPU die target and a 111 C SoC-hot temperature), because the sensors may
+//! read cooler than the GPU's hottest spot (the PMP's are CPU cluster sensors), and this limit
+//! reacts in steps of hundreds of milliseconds while the firmware changes state within the cap
+//! much faster.
+//!
+//! The cap is changed only by the job thread, after a job has retired, right before the state
+//! the firmware reports is checked. After a lowering the previous cap is tolerated for 1 s. A
+//! reported state above the runtime cap in 20 consecutive checks spanning at least 3 s means
+//! the firmware does not follow the runtime cap, so this limit cannot work: the GPU is then
+//! marked failed, as for a state above the ceiling.
+//!
+//! The decisions are in `m3_thermal_policy`, which host tests drive; this module reads the
+//! sensors, publishes the cap and logs.
+
+use core::sync::atomic::{AtomicBool, Ordering};
+
+use kernel::{
+    c_str, device,
+    error::to_result,
+    new_spinlock,
+    prelude::*,
+    str::CStr,
+    sync::{Arc, SpinLock},
+    time::{msecs_to_jiffies, Instant, Monotonic},
+    workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem},
+};
+
+use crate::{
+    m3::m3_adt_config::PstatePolicy,
+    m3::m3_params::{self, ThermalMode},
+    m3::m3_thermal_policy::{Change, Policy, Reading, Source, Why, S},
+};
+
+/// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors
+/// (registered by the SMC hwmon driver) and the PMP's CPU cluster sensors (registered by the
+/// PMP report driver once the PMP runs). Zones that do not exist are skipped; a zone that
+/// exists but cannot be read fails the reading. The M3 Pro boards (J514S and J516S) need
+/// both zones: on a J516S the SMC die sensors alone stayed below 67 C while the PMP hotspot
+/// exceeded 85 C and the cap stepped down. Without the PMP running, the cap stays safe.
+/// Every T8122 board requires the SMC die zone; T8122 has no PMP hotspot zone.
+const ZONES: [&CStr; 2] = [c_str!("macsmc_soc_die"), c_str!("apple_pmp_hotspot")];
+// Do not raise its cap on a partial reconstruction of that sensor set.
+const J516_REQUIRED_ZONES: u32 = 3;
+fn required_zones() -> u32 {
+    let Some(root) = kernel::of::root() else { return J516_REQUIRED_ZONES; };
+    let Ok(compatible) = root.get_property::<KVec<u8>>(c_str!("compatible")) else {
+        return J516_REQUIRED_ZONES;
+    };
+    crate::m3::m3_thermal_policy::required_zones(&compatible)
+}
+
+/// Sensor polling period while the GPU is busy and while it is idle, in ms.
+const POLL_BUSY_MS: u32 = 500;
+const POLL_IDLE_MS: u32 = 2000;
+/// The GPU counts as busy for this long after a job has retired, in ns.
+const BUSY_WINDOW_NS: i64 = 5 * S;
+/// Lines logging cap changes per boot, and the period of the summary line while busy.
+const LOG_LINES: u32 = 256;
+const SUMMARY_EVERY_NS: i64 = 30 * S;
+
+type Time = Instant<Monotonic>;
+
+/// Nanoseconds from `base` to `t`.
+fn ns(base: Time, t: Time) -> i64 {
+    (t - base).as_nanos()
+}
+
+struct SensorState {
+    reading: Reading,
+    /// When the GPU last retired a job, in ns from the sensor's base.
+    busy: i64,
+}
+
+/// Reads [`ZONES`] from a self-rearming delayed work item, so the job thread never waits for the
+/// SMC. The work item holds a reference, so it stays valid after the governor is dropped; it
+/// then stops at its next run.
+#[pin_data]
+struct Sensor {
+    required_zones: u32,
+    /// The origin of the sensor's and the governor's times.
+    base: Time,
+    stopped: AtomicBool,
+    #[pin]
+    state: SpinLock<SensorState>,
+    #[pin]
+    work: DelayedWork<Sensor>,
+}
+
+impl_has_delayed_work! {
+    impl HasDelayedWork<Self> for Sensor { self.work }
+}
+
+extern "C" {
+    // include/linux/thermal.h; not in 7.2's generated bindings.
+    fn thermal_zone_get_temp_by_name(name: *const core::ffi::c_char, temp: *mut i32) -> core::ffi::c_int;
+}
+
+/// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone.
+fn read_zone(name: &CStr) -> Result<i32> {
+    let mut temp: i32 = 0;
+    // SAFETY: name is NUL-terminated and temp is a valid output pointer. The
+    // thermal core acquires a device reference under its registry lock and
+    // keeps it through sampling, so zone unregistration cannot free it here.
+    to_result(unsafe { thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?;
+    Ok(temp)
+}
+
+/// The hottest reading of the zones in [`ZONES`] that exist, and which ones were read.
+fn read_zones(required: u32) -> Result<(i32, u32)> {
+    let mut hottest: Option<i32> = None;
+    let mut zones = 0;
+    for (i, name) in ZONES.iter().enumerate() {
+        match read_zone(name) {
+            Ok(temp) => {
+                hottest = Some(hottest.map_or(temp, |h| h.max(temp)));
+                zones |= 1 << i;
+            }
+            Err(e) if e == ENODEV => {}
+            Err(e) => return Err(e),
+        }
+    }
+    if zones & required != required {
+        return Err(ENODEV);
+    }
+    hottest.map(|t| (t, zones)).ok_or(ENODEV)
+}
+
+impl WorkItem for Sensor {
+    type Pointer = Arc<Sensor>;
+
+    fn run(this: Arc<Sensor>) {
+        if this.stopped.load(Ordering::Acquire) {
+            return;
+        }
+        let result = read_zones(this.required_zones);
+        let at = ns(this.base, Time::now());
+        let reading = match result {
+            Ok((mc, zones)) => Reading::Temp { mc, at, zones },
+            Err(e) => Reading::Failed {
+                errno: e.to_errno(),
+                at,
+            },
+        };
+        let busy = {
+            let mut state = this.state.lock();
+            state.reading = reading;
+            at - state.busy < BUSY_WINDOW_NS
+        };
+        let delay = if busy { POLL_BUSY_MS } else { POLL_IDLE_MS };
+        let _ = workqueue::system().enqueue_delayed(this, msecs_to_jiffies(delay));
+    }
+}
+
+fn describe(source: Source) -> &'static str {
+    match source {
+        Source::Unknown | Source::NoReading => "no reading yet",
+        Source::Ok(_) => "reading",
+        Source::Failed(e) if e == ENODEV.to_errno() => "no SoC die temperature zone",
+        Source::Failed(_) => "a SoC die temperature zone cannot be read",
+        Source::Stale => "the reading is older than 3 s",
+        Source::Implausible => "the reading is outside 10..=125 C",
+        Source::Stuck => "the reading has not changed for 60 s",
+    }
+}
+
+fn describe_why(why: Why) -> &'static str {
+    match why {
+        Why::Critical => "critical",
+        Why::Hot => "hot",
+        Why::Cool => "cool",
+        Why::NoTemperature(source) => describe(source),
+    }
+}
+
+/// Degrees Celsius with one decimal, for the log.
+struct Celsius(i32);
+
+impl kernel::fmt::Display for Celsius {
+    fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        let sign = if self.0 < 0 { "-" } else { "" };
+        let v = self.0.unsigned_abs();
+        write!(f, "{}{}.{} C", sign, v / 1000, (v % 1000) / 100)
+    }
+}
+
+/// The runtime cap and the thermal limit that moves it.
+pub(crate) struct Governor {
+    mode: ThermalMode,
+    /// Without the limit (`off`): the cap, which never changes.
+    fixed: u32,
+    policy: Policy,
+    freqs: [u32; 16],
+    base: Time,
+    sensor: Option<Arc<Sensor>>,
+    source: Source,
+    last_summary: Option<i64>,
+    /// The highest state the firmware reported since the last summary.
+    peak: u32,
+    lines: u32,
+}
+
+impl Governor {
+    /// The governor for `policy`. Starts the sensor work item unless the limit is off.
+    pub(crate) fn new(dev: &device::Device, pstates: &PstatePolicy) -> Result<Self> {
+        let (hot_c, clamped) = m3_params::thermal_hot_param();
+        let base = Time::now();
+        let mut governor = Governor {
+            mode: pstates.thermal,
+            fixed: pstates.max,
+            policy: Policy::new(
+                pstates.thermal == ThermalMode::On,
+                pstates.safe,
+                pstates.max,
+                hot_c as i32 * 1000,
+            ),
+            freqs: pstates.freqs,
+            base,
+            sensor: None,
+            source: Source::Unknown,
+            last_summary: None,
+            peak: 0,
+            lines: 0,
+        };
+        if governor.mode == ThermalMode::Off {
+            return Ok(governor);
+        }
+        if clamped {
+            dev_info!(
+                dev,
+                "M3: asahi.m3_thermal_hot clamped to {} C ({}..={} C)\n",
+                hot_c,
+                m3_params::THERMAL_HOT_MIN,
+                m3_params::THERMAL_HOT_MAX
+            );
+        }
+        let sensor = Arc::pin_init(
+            pin_init!(Sensor {
+                required_zones: required_zones(),
+                base,
+                stopped: AtomicBool::new(false),
+                state <- new_spinlock!(SensorState {
+                    reading: Reading::None,
+                    busy: 0,
+                }),
+                work <- new_delayed_work!("asahi::M3ThermalSensor"),
+            }),
+            GFP_KERNEL,
+        )?;
+        let _ = workqueue::system().enqueue_delayed(sensor.clone(), msecs_to_jiffies(POLL_BUSY_MS));
+        governor.sensor = Some(sensor);
+        let p = &governor.policy;
+        let (cool, hot, critical) = p.thresholds();
+        match governor.mode {
+            ThermalMode::Hold => dev_info!(
+                dev,
+                "M3: thermal limit holding (asahi.m3_thermal=hold): the firmware may use states 1..={} ({} MHz) and the runtime cap stays at {} ({} MHz) for this boot; the SoC die temperature is logged only\n",
+                p.ceiling(),
+                governor.mhz(p.ceiling()),
+                p.safe(),
+                governor.mhz(p.safe())
+            ),
+            _ => dev_info!(
+                dev,
+                "M3: thermal limit on (asahi.m3_thermal=on): runtime cap {}..={} ({}..={} MHz) from the hottest SoC die temperature of zones macsmc_soc_die and apple_pmp_hotspot: up while at or below {}, down at or above {}, cap {} at or above {} or without a fresh reading\n",
+                p.safe(),
+                p.ceiling(),
+                governor.mhz(p.safe()),
+                governor.mhz(p.ceiling()),
+                Celsius(cool),
+                Celsius(hot),
+                p.safe(),
+                Celsius(critical)
+            ),
+        }
+        Ok(governor)
+    }
+
+    fn mhz(&self, state: u32) -> u32 {
+        self.freqs.get(state as usize).copied().unwrap_or(0)
+    }
+
+    fn cap(&self) -> u32 {
+        match self.mode {
+            ThermalMode::Off => self.fixed,
+            ThermalMode::Hold | ThermalMode::On => self.policy.cap(),
+        }
+    }
+
+    fn note_source(&mut self, dev: &device::Device, source: Source, temp: Option<i32>) {
+        if source == self.source {
+            return;
+        }
+        self.source = source;
+        let safe = self.policy.safe();
+        match (source, temp) {
+            (Source::Ok(zones), Some(t)) => {
+                let smc = if zones & 1 != 0 {
+                    " macsmc_soc_die (SMC die sensors)"
+                } else {
+                    ""
+                };
+                let pmp = if zones & 2 != 0 {
+                    " apple_pmp_hotspot (PMP CPU cluster sensors)"
+                } else {
+                    ""
+                };
+                dev_info!(
+                    dev,
+                    "M3: thermal: SoC die temperature {} from zones:{}{}\n",
+                    Celsius(t),
+                    smc,
+                    pmp
+                )
+            }
+            (Source::Failed(e), _) => dev_info!(
+                dev,
+                "M3: thermal: {} (error {}); runtime cap at most {} ({} MHz)\n",
+                describe(source),
+                e,
+                safe,
+                self.mhz(safe)
+            ),
+            _ => dev_info!(
+                dev,
+                "M3: thermal: {}; runtime cap at most {} ({} MHz)\n",
+                describe(source),
+                safe,
+                self.mhz(safe)
+            ),
+        }
+    }
+
+    fn summary(&mut self, dev: &device::Device, now: i64, temp: Option<i32>) {
+        if self
+            .last_summary
+            .is_some_and(|t| now - t < SUMMARY_EVERY_NS)
+        {
+            return;
+        }
+        self.last_summary = Some(now);
+        let peak = core::mem::take(&mut self.peak);
+        let cap = self.policy.cap();
+        let (raised, lowered) = self.policy.counts();
+        let (safe, ceiling) = (self.policy.safe(), self.policy.ceiling());
+        match temp {
+            Some(t) => dev_info!(
+                dev,
+                "M3: thermal: SoC die {}, runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n",
+                Celsius(t),
+                cap,
+                self.mhz(cap),
+                safe,
+                ceiling,
+                peak,
+                raised,
+                lowered
+            ),
+            None => dev_info!(
+                dev,
+                "M3: thermal: no SoC die temperature ({}), runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n",
+                describe(self.source),
+                cap,
+                self.mhz(cap),
+                safe,
+                ceiling,
+                peak,
+                raised,
+                lowered
+            ),
+        }
+    }
+
+    fn log_change(&mut self, dev: &device::Device, change: Change, temp: Option<i32>) {
+        if self.lines < LOG_LINES {
+            match temp {
+                Some(t) => dev_info!(
+                    dev,
+                    "M3: thermal: runtime cap {} -> {} ({} MHz): SoC die {} ({})\n",
+                    change.from,
+                    change.to,
+                    self.mhz(change.to),
+                    Celsius(t),
+                    describe_why(change.why)
+                ),
+                None => dev_info!(
+                    dev,
+                    "M3: thermal: runtime cap {} -> {} ({} MHz): {}\n",
+                    change.from,
+                    change.to,
+                    self.mhz(change.to),
+                    describe_why(change.why)
+                ),
+            }
+        } else if self.lines == LOG_LINES {
+            dev_info!(
+                dev,
+                "M3: thermal: further runtime cap changes are not logged\n"
+            );
+        }
+        self.lines = self.lines.saturating_add(1);
+    }
+
+    /// Recompute the runtime cap after a job. Returns the new cap if it changed; the caller
+    /// publishes it.
+    pub(crate) fn update(&mut self, dev: &device::Device, now: Time) -> Option<u32> {
+        let Some(sensor) = &self.sensor else {
+            return None;
+        };
+        let reading = sensor.state.lock().reading;
+        let now = ns(self.base, now);
+        let (usable, change) = self.policy.update(now, reading);
+        let temp = usable.ok().map(|(t, _, _)| t);
+        match usable {
+            Err(source) => self.note_source(dev, source, None),
+            Ok((t, _, zones)) => self.note_source(dev, Source::Ok(zones), Some(t)),
+        }
+        self.summary(dev, now, temp);
+        let change = change?;
+        self.log_change(dev, change, temp);
+        Some(change.to)
+    }
+
+    /// The highest state the firmware may report now: the runtime cap, or the highest cap of
+    /// the grace period after a lowering.
+    pub(crate) fn allowed(&self, now: Time) -> u32 {
+        match self.mode {
+            ThermalMode::Off => self.fixed,
+            ThermalMode::Hold | ThermalMode::On => self.policy.allowed(ns(self.base, now)),
+        }
+    }
+
+    /// Account one check of the reported state; `over` says whether it stayed above
+    /// [`Self::allowed`], `peak` is the highest state it reported. Fails with ERANGE when the
+    /// firmware does not follow the runtime cap.
+    pub(crate) fn note_check(
+        &mut self,
+        dev: &device::Device,
+        now: Time,
+        over: bool,
+        peak: u32,
+    ) -> Result {
+        self.peak = self.peak.max(peak);
+        if let Some((count, span)) = self.policy.note_check(ns(self.base, now), over) {
+            let cap = self.cap();
+            dev_err!(
+                dev,
+                "M3: thermal: the firmware reported a state above the runtime cap {} ({} MHz) in {} consecutive checks over {} ms: it does not follow the runtime cap; marking the GPU failed\n",
+                cap,
+                self.mhz(cap),
+                count,
+                span / 1_000_000
+            );
+            return Err(ERANGE);
+        }
+        Ok(())
+    }
+
+    /// Note that the GPU retired a job, so the sensor is read at the busy rate.
+    pub(crate) fn busy(&self, now: Time) {
+        if let Some(sensor) = &self.sensor {
+            sensor.state.lock().busy = ns(self.base, now);
+        }
+    }
+}
+
+impl Drop for Governor {
+    fn drop(&mut self) {
+        if let Some(sensor) = &self.sensor {
+            sensor.stopped.store(true, Ordering::Release);
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs
index eeda857a3ed88..caa61621e0b04 100644
--- a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs
+++ b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs
@@ -3,6 +3,17 @@ use super::{agx_render::Geometry, m3_compute_layout::{GpuVa,RegisterSet},
     m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa,
     m3_render_sequence::ta};
 pub(crate) const SIZE:usize=0x92b;
+pub(crate) const EXTENDED_SIZE:usize=0x940;
+pub(crate) const EXTENDED_FLUSH:usize=0x894;
+pub(crate) struct Dispatch26 {
+    pub(crate) incarnation:u8,
+    pub(crate) serial:u64,
+    pub(crate) peer_stamp:u32,
+    pub(crate) usc_window:u64,
+    pub(crate) firmware_aux:FirmwareVa,
+    pub(crate) scene_scratch:GpuVa,
+}
+
 pub(crate) const REGISTERS:usize=ta::REGISTERS;
 /// The registers G15G does not take; the others keep the G15S order.
 pub(crate) const G15G_OMIT:[u32;14]=[0x1c041,0x1c9d0,0x1c948,0x1c888,0x1c890,0x1c918,0x1c089,0x1c9e0,0x16c41,0x1ca40,0x1c9a8,0x1c920,0xa5a1,0xd419];
@@ -57,17 +68,32 @@ impl Command {
     /// All address checks precede mutation. Zero all reserved fields, then
     /// emit semantic registers and explicit packed command/tail fields.
     pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> {
-        if out.len()<SIZE {return Err(Error::Size);}
+        self.encode_format(out, None)
+    }
+    pub(crate) fn encode_extended(&self, dispatch: Dispatch26, out: &mut [u8]) -> Result<(),Error> {
+        self.encode_format(out, Some(dispatch))
+    }
+    fn encode_format(&self, out: &mut [u8], dispatch: Option<Dispatch26>) -> Result<(),Error> {
+        let extended = dispatch.is_some();
+        let size = if extended { EXTENDED_SIZE } else { SIZE };
+        if out.len()<size {return Err(Error::Size);}
+        if let Some(native) = &dispatch {
+            if native.incarnation == 0 || native.serial == 0 || native.peer_stamp == 0 || native.usc_window & 0xffff_ffff != 0
+                || native.usc_window >= 1 << 42 || self.register_set != RegisterSet::G15G {
+                return Err(Error::Address);
+            }
+        }
         if self.context>=64 || self.page_count==0 || self.tpc_bytes==0 {return Err(Error::Address);}
         let registers=self.gpu_alias.get().checked_add(REGISTERS as u64).ok_or(Error::Address)?;
-        GpuVa::new(self.gpu_alias.get().checked_add((SIZE-3) as u64).ok_or(Error::Address)?)
+        GpuVa::new(self.gpu_alias.get().checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?)
             .map_err(|_|Error::Address)?;
-        let seq=self.sequence.at(0,ta::LENGTH)?;
+        let sequence_size = if extended { 0x300 } else { ta::LENGTH };
+        let seq=self.sequence.at(0,sequence_size)?;
         let tilemap=compact(self.tilemap.get())?;
         let tpc=compact(self.tpc.get())?;
         let heap=compact(self.heap.get())?;
         let preemption=[compact(self.preemption[0].get())?,compact(self.preemption[1].get())?,compact(self.preemption[2].get())?];
-        let user=compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)?;
+        let user=compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)?;
         let g=&self.geometry;
         let layers=g.layers;
         let layer_metadata=compact(self.heap.get().checked_add(if layers>1 {0x200}else{0}).ok_or(Error::Address)?)?;
@@ -99,23 +125,67 @@ impl Command {
             (0x1c830,0),(0x1ca30,user),(0x16c39,user),
             (0x1c910,0xa0_00000000|(self.initial_scene_entry.get()&0xf_ffffffff)|1),
         ];
-        let bytes=&mut out[..SIZE];bytes.fill(0); // TA opcode is zero
-        pointer(bytes,4,1); // qualified work-command counter remains one
+        let bytes=&mut out[..size];bytes.fill(0); // TA opcode is zero
+        pointer(bytes,4,dispatch.as_ref().map_or(1, |native| native.serial));
         word(bytes,CONTEXT,self.context);
-        for (offset,value) in [(0x14,self.notifier.get()),(0x24,self.manager.get()),
-            (0x2c,self.scene.get()),(0x34,self.empty.get()),(0x740,registers),
+        for (offset,value) in [(if extended {0x10} else {0x14},self.notifier.get()),
+            (if extended {0x20} else {0x24},self.manager.get()),
+            (if extended {0x28} else {0x2c},self.scene.get()),
+            (if extended {0x30} else {0x34},self.empty.get()),(0x740,registers),
             (0x760,self.tpc.get()),(0x768,self.tpc_bytes),(0x770,seq)] {pointer(bytes,offset,value);}
         let g15g=self.register_set==RegisterSet::G15G;
         let mut count=0;
-        for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) {
-            let n=count;count+=1;
-            word(bytes,REGISTERS+n*REGISTER_STRIDE,register);
-            pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value);
+        let mut emit = |register, value| {
+            word(bytes, REGISTERS + count * REGISTER_STRIDE, register);
+            pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value);
+            count += 1;
+        };
+        for (register, mut value) in values {
+            if g15g && G15G_OMIT.contains(&register) { continue; }
+            if extended && matches!(register, 0x12099 | 0x1c898) { continue; }
+            if let Some(native) = &dispatch {
+                if register == 0x101e1 {
+                    emit(0xa309, 0);
+                    emit(0x1c8e0, u64::MAX);
+                    emit(0x1c8e8, u64::MAX);
+                    emit(0x1c898, 0);
+                }
+                if register == 0x1c910 {
+                    let address = native.scene_scratch.get();
+                    let escaped = (address & (1 << 42)) >> 3;
+                    let bias = if escaped == 0 { 0x70_0000_0000 } else { 0 };
+                    value = ((address + bias) & 0x7f_ffff_fffe) | escaped | 1;
+                }
+            }
+            emit(register, value);
+        }
+        if let Some(native) = &dispatch {
+            emit(0xa5a1, 0x60_0040_0020);
+            emit(0xd419, 0x2_0000_0001);
+            for register in [0x1ca10, 0x14a1, 0xa349] { emit(register, ta::UUID.into()); }
+            emit(0x10061, native.usc_window);
         }
         bytes[0x748..0x74a].copy_from_slice(&(count as u16).to_le_bytes());
         bytes[0x74a..0x74c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes());
-        word(bytes,SEQUENCE_LENGTH,ta::LENGTH as u32);
+        word(bytes,SEQUENCE_LENGTH,sequence_size as u32);
         word(bytes,0x77c,1); // fragment event slot
+        if let Some(native) = dispatch {
+            word(bytes, STAMP_VALUE, native.peer_stamp);
+            pointer(bytes, 0x7a8, preemption[0]);
+            pointer(bytes, 0x878, self.stamp.get());
+            pointer(bytes, 0x880, self.fw_stamp.get());
+            word(bytes, 0x888, self.stamp_value);
+            word(bytes, EXTENDED_FLUSH, 1);
+            word(bytes, 0x898, ta::UUID);
+            pointer(bytes, 0x8c8, self.timestamps[0].get());
+            pointer(bytes, 0x8d0, self.timestamps[1].get());
+            pointer(bytes, 0x8d8, self.user_timestamps[0].get());
+            pointer(bytes, 0x8e0, self.user_timestamps[1].get());
+            pointer(bytes, 0x8fe, self.pool.get());
+            bytes[0x906] = 1;
+            pointer(bytes, 0x90f, native.firmware_aux.get());
+            bytes[0x917] = native.incarnation;
+        } else {
         word(bytes,STAMP_VALUE,self.stamp_value);
         pointer(bytes,ta::STRUCT3+0x24,preemption[0]);
         word(bytes,ta::STRUCT3+0xbc,0x67); // retained producer encoder_id
@@ -132,6 +202,41 @@ impl Command {
             (0x47,self.scratch.get())] {pointer(bytes,ta::COMMAND_TAIL+offset,value);}
         bytes[ta::COMMAND_TAIL+0x36]=1; // unk_846
         bytes[ta::COMMAND_TAIL+0x4f]=1; // unk_85f
+        }
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod extended_tests {
+    use super::*;
+    fn command() -> Command {
+        let fw=|index:u64|FirmwareVa::new(0xffff_fc25_1000_0000+index*0x4000).unwrap();
+        let gpu=|index:u64|GpuVa::new(0x10_0100_0000+index*0x4000).unwrap();
+        Command {context:3,register_set:RegisterSet::G15G,notifier:fw(0),manager:fw(1),scene:fw(2),empty:fw(3),
+            gpu_alias:GpuVa::new(0x72_1000_0000).unwrap(),sequence:Region::new(fw(4).get(),0x1000).unwrap(),
+            pool:fw(5),scratch:fw(6),stamp:fw(7),fw_stamp:fw(8),stamp_value:0x7a000100,
+            timestamps:[fw(9),fw(10)],user_timestamps:[fw(11),fw(12)],preemption:[gpu(0),gpu(1),gpu(2)],
+            tilemap:gpu(3),tpc:gpu(4),tpc_bytes:0x4000,heap:gpu(5),scene_user:gpu(6),
+            initial_scene_entry:fw(13),geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32,
+                super::super::agx_render::Utile::Pixels32,1).unwrap(),page_count:128,
+            vdm:0x100000,multisample:0,ppp:0,process_empty_tiles:false}
+    }
+    #[test] fn native_tiler_keeps_peer_completion_and_usc_window_explicit() {
+        let value=command();let mut bytes=[0xa5;EXTENDED_SIZE+8];
+        value.encode_extended(Dispatch26 {incarnation:9,serial:2,peer_stamp:0x3d000100,
+            usc_window:1<<32,firmware_aux:value.scratch,
+            scene_scratch:GpuVa::new(0x10_0200_0000).unwrap()},&mut bytes).unwrap();
+        let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap());
+        assert_eq!(word(0x780),0x3d000100);
+        assert_eq!(word(0x888),value.stamp_value);
+        assert_eq!(word(0x894),1);
+        assert_eq!(word(0x748)&0xffff,66);
+        assert_eq!(bytes[0x917],9);
+        assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]);
+        let registers=&bytes[REGISTERS..REGISTERS+66*REGISTER_STRIDE];
+        assert!(!registers.chunks_exact(12).any(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x12099));
+        let usc=registers.chunks_exact(12).find(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x10061).unwrap();
+        assert_eq!(u64::from_le_bytes(usc[4..].try_into().unwrap()),1<<32);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs
index 3d900a80fef38..eafd5732bceac 100644
--- a/drivers/gpu/drm/asahi/m3/mod.rs
+++ b/drivers/gpu/drm/asahi/m3/mod.rs
@@ -43,6 +43,98 @@ pub(crate) fn refuse_probe(pdev: &platform::Device<Core<'_>>, chip: Chip) -> Err
     ENODEV
 }
 
+/// Whether the M3 runtime may start on this T8122 (M3, G15G), whose GPU the boot loader handed
+/// over: its identity gate passes (only DT properties are read) and the backend selection allows
+/// the board. The runtime then admits the boot loader's description, and refuses the SoC unless
+/// `asahi.t8122_start=1` arms the start ([`t8122_start`]).
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+pub(crate) fn t8122_admitted(pdev: &platform::Device<Core<'_>>) -> Result<&'static m3_soc::Soc> {
+    let identity = t8122_admission::read_identity(pdev).map_err(|error| {
+        dev_info!(
+            pdev.as_ref(),
+            "G15G: T8122 identity or ABI rejected ({:?}); startup refused\n",
+            error
+        );
+        ENODEV
+    })?;
+    dev_info!(
+        pdev.as_ref(),
+        "G15G: T8122 revision 2.0 identity admitted ({} active / {} slots)\n",
+        identity.active_cores,
+        identity.core_slots
+    );
+    if !m3_params::t8122_backend(pdev) {
+        return Err(ENODEV);
+    }
+    Ok(&m3_soc::T8122)
+}
+
+// The T8122 runtime, ported from the 7.1.12 release-20 kernel (c46d7dcdb871).
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_attachments;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_compute;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_memory;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_resources;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_status;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod agx_uapi;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod g15_raw;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod hw;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod identity;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod initdata;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_adt_config;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_board;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_client;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_completion;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_compute;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_config;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_device;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+pub(crate) mod m3_drm;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_firmware;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_memory;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_params;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_pass;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_pool;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_render;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_resources;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_rtkit;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_runtime;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_submit;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_thermal;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod t8122_knobs;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod t8122_start;
+
+mod m3_completion_policy;
+
 mod m3_compute_layout;
 
 mod m3_compute_sequence;
@@ -103,6 +195,17 @@ mod m3_fragment_command;
 
 mod t8122_admission;
 
-mod m3_soc;
+// The runtime's SoC tables; they name the runtime's own modules.
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+pub(crate) mod m3_soc;
 
 mod t8122_hwdata;
+
+mod m3_init_26;
+mod m3_profile;
+mod m3_mmio_26;
+
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3_power_26;
+
+mod m3_channel_format;
diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs
index 12bea894c373e..fb280ebcde520 100644
--- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs
+++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
-//! Read-only T8122 identity admission. Firmware startup remains disabled.
+//! Read-only T8122 identity and firmware-interface admission.
+
+use super::m3_profile::Profile;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub(crate) struct Identity {
@@ -16,12 +18,23 @@ pub(crate) fn validate_identity(
     result: u32,
     id_version: u32,
     masks: [u32; 3],
+) -> Option<Identity> {
+    validate_identity_for(Profile::Legacy, firmware_compat, schema, result, id_version, masks)
+}
+
+pub(crate) fn validate_identity_for(
+    profile: Profile,
+    firmware_compat: &[u32],
+    schema: u32,
+    result: u32,
+    id_version: u32,
+    masks: [u32; 3],
 ) -> Option<Identity> {
     let family = id_version >> 24;
     let variant = (id_version >> 16) & 0xff;
     let revision = (id_version >> 8) & 0xff;
     let active_cores = masks[0].count_ones();
-    if firmware_compat != [14, 8, 3]
+    if firmware_compat != profile.firmware_compat()
         || schema != 1
         || result != 1
         || family != 7
@@ -40,6 +53,28 @@ pub(crate) fn validate_identity(
     })
 }
 
+/// Firmware-interface selection is exact and does not change physical GPU encodings.
+pub(crate) fn profile_for(compat:&[u32],j613:bool)->Option<Profile> {
+    match compat {
+        [14,8,3]=>Some(Profile::Legacy),
+        [26,6,2] if j613=>Some(Profile::Extended),
+        _=>None,
+    }
+}
+
+#[cfg(not(test))]
+pub(crate) fn loaded_profile(pdev:&kernel::platform::Device<kernel::device::Core<'_>>)
+    ->kernel::error::Result<Profile> {
+    use kernel::{c_str,prelude::*};
+    let node=pdev.as_ref().of_node().ok_or(ENODEV)?;
+    let compat:KVec<u32>=node.get_property(c_str!("apple,firmware-compat"))?;
+    #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+    let j613 = super::m3_board::board_is(b"apple,j613");
+    #[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))]
+    let j613 = false;
+    profile_for(&compat, j613).ok_or(ENODEV)
+}
+
 /// Shape of the voltage-sorted running-state tables: one entry per distinct millivolt,
 /// and the highest primary frequency in MHz. Voltages round up, matching `PwrConfig`.
 /// The first point is the off state. Both primary and secondary frequencies must increase
@@ -94,6 +129,14 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator<Item = (u64, u32)>) -> Opt
 #[cfg(not(test))]
 pub(crate) fn read_identity(
     pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+) -> kernel::error::Result<Identity> {
+    read_identity_for(pdev, loaded_profile(pdev)?)
+}
+
+#[cfg(not(test))]
+pub(crate) fn read_identity_for(
+    pdev: &kernel::platform::Device<kernel::device::Core<'_>>,
+    profile: Profile,
 ) -> kernel::error::Result<Identity> {
     use kernel::{bindings, c_str, prelude::*};
 
@@ -146,7 +189,8 @@ pub(crate) fn read_identity(
         observation.get_u32(c_str!("core-mask-1"))?,
         observation.get_u32(c_str!("core-mask-2"))?,
     ];
-    validate_identity(firmware_compat.as_slice(), schema, result, id_version, masks).ok_or(ENODEV)
+    validate_identity_for(profile, firmware_compat.as_slice(), schema, result, id_version, masks)
+        .ok_or(ENODEV)
 }
 
 #[cfg(test)]
@@ -164,6 +208,29 @@ mod tests {
         }
     }
 
+    #[test]
+    fn extended_identity_requires_its_actual_version_and_the_same_g15g_bounds() {
+        for mask in [0xff, 0x3ff] {
+            let good = || validate_identity_for(Profile::Extended, &[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]);
+            assert_eq!(good().unwrap().active_cores, mask.count_ones());
+            assert!(validate_identity(&[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]).is_none());
+            assert!(validate_identity_for(Profile::Extended, &[14, 8, 3], 1, 1, G15G_C0, [mask, 0, 0]).is_none());
+        }
+        for (schema, result, id, masks) in [
+            (0, 1, G15G_C0, [0xff, 0, 0]),
+            (1, 0, G15G_C0, [0xff, 0, 0]),
+            (1, 1, G15G_C0 ^ (1 << 24), [0xff, 0, 0]),
+            (1, 1, G15G_C0 ^ (1 << 16), [0xff, 0, 0]),
+            (1, 1, G15G_C0 ^ (1 << 8), [0xff, 0, 0]),
+            (1, 1, G15G_C0, [0x1ff, 0, 0]),
+            (1, 1, G15G_C0, [0x7ff, 0, 0]),
+            (1, 1, G15G_C0, [0xff, 1, 0]),
+            (1, 1, G15G_C0, [0xff, 0, 1]),
+        ] {
+            assert!(validate_identity_for(Profile::Extended, &[26, 6, 2], schema, result, id, masks).is_none());
+        }
+    }
+
     /// The booted J613 ladder (`perf-states`, frequency then millivolts), in ADT order: fourteen
     /// states, eight voltages above the idle one, 1338 MHz at the top.
     const J613_PERF_STATES: [(u64, u32); 14] = [
@@ -255,3 +322,17 @@ mod tests {
         assert!(validate_identity(&[14, 8, 3], 1, 1, G15G_C0, [0x7f, 0, 0]).is_none());
     }
 }
+
+#[cfg(test)]
+mod selection_tests {
+    use super::*;
+    #[test] fn extended_admission_requires_exact_firmware_and_j613() {
+        for board in [false,true] {
+            assert_eq!(profile_for(&[14,8,3],board),Some(Profile::Legacy));
+            assert_eq!(profile_for(&[26,6,2],board),if board{Some(Profile::Extended)}else{None});
+            for wrong in [&[][..],&[26,6][..],&[26,6,1][..],&[26,6,2,0][..],&[14,8,4][..]] {
+                assert_eq!(profile_for(wrong,board),None);
+            }
+        }
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs
new file mode 100644
index 0000000000000..ca0f8a6d499cc
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs
@@ -0,0 +1,357 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! The parameters of the T8122 start experiment (`t8122_start`): their text, defaults, accepted
+//! values and their meaning. Plain data only, so that the host tests run it (tools/asahi).
+
+/// A parameter that is not given.
+pub(crate) const UNSET: u64 = u64::MAX;
+/// A parameter whose text is not one of its accepted values. The experiment refuses to start
+/// with it rather than fall back to a default.
+pub(crate) const INVALID: u64 = u64::MAX - 1;
+
+/// The Fender window sizes: the ADT sgx reg[1] size less 0x28000 (the difference on T6030), and
+/// the whole ADT sgx reg[1].
+pub(crate) const FENDER_RULE: u32 = 0x10_4000;
+pub(crate) const FENDER_ADT: u32 = 0x12_c000;
+/// The GPU clock-generator offsets in SGX: the T8122 table's (`T8122_IO_MAPPINGS`, the default),
+/// and the T6030 one.
+pub(crate) const CLOCK_GEN_E1C: u64 = 0xe1_c000;
+pub(crate) const CLOCK_GEN_E5C: u64 = 0xe5_c000;
+/// The SGX setup write of T6030 and T8122 (offset, value).
+pub(crate) const T6030_SGX_SETUP: (usize, u32) = (0xd1_4000, 0x7_0001);
+/// The default unit masks: the T8122 table's (`m3_soc::T8122_HWDATA_B`), which are T6030's.
+pub(crate) const UNIT_MASK_A: u64 = 0x7_0000_0003;
+pub(crate) const UNIT_MASK_B: u32 = 0x7;
+/// No unit-mask bit outside these is accepted: the default masks.
+pub(crate) const UNIT_MASK_A_LIMIT: u64 = 0x7_0000_0003;
+pub(crate) const UNIT_MASK_B_LIMIT: u64 = 0x7;
+/// The default performance-state ceiling: the lowest two states.
+pub(crate) const PSTATE_CAP: u32 = 2;
+/// The highest performance-state ceiling the experiment accepts. The same as the driver's own
+/// limit when the fast-die temperature controller is off (`m3_adt_config::ADT_MAX_PSTATE_LIMIT`):
+/// J613/J615 have no SoC die temperature zone yet, so there is no thermal feedback above it.
+pub(crate) const PSTATE_CAP_MAX: u32 = 5;
+
+/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. A leading sign is not accepted.
+fn number(text: &str) -> Option<u64> {
+    if text.starts_with(['+', '-']) {
+        return None;
+    }
+    match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) {
+        Some(hex) if !hex.starts_with(['+', '-']) => u64::from_str_radix(hex, 16).ok(),
+        Some(_) => None,
+        None => text.parse::<u64>().ok(),
+    }
+}
+
+/// Parse a parameter's text: one of `names`, else a number. Anything else, and the two
+/// reserved values, give [`INVALID`].
+fn parse(text: &str, names: &[(&str, u64)]) -> u64 {
+    let text = text.trim();
+    if let Some(&(_, value)) = names.iter().find(|(name, _)| *name == text) {
+        return value;
+    }
+    number(text).filter(|v| *v < INVALID).unwrap_or(INVALID)
+}
+
+pub(crate) fn parse_number(text: &str) -> u64 {
+    parse(text, &[])
+}
+
+pub(crate) fn parse_fender(text: &str) -> u64 {
+    parse(text, &[("rule", FENDER_RULE as u64), ("adt", FENDER_ADT as u64)])
+}
+
+pub(crate) fn parse_clkgen(text: &str) -> u64 {
+    parse(text, &[("e1c", CLOCK_GEN_E1C), ("e5c", CLOCK_GEN_E5C), ("none", 0)])
+}
+
+pub(crate) fn parse_sgx_setup(text: &str) -> u64 {
+    parse(text, &[("none", 0), ("t6030", 1)])
+}
+
+/// The parameters as given: [`UNSET`], [`INVALID`] or a value.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) struct Raw {
+    pub(crate) start: u64,
+    pub(crate) initdata_version: u64,
+    pub(crate) fender: u64,
+    pub(crate) clkgen: u64,
+    pub(crate) sgx_setup: u64,
+    pub(crate) unit_mask_a: u64,
+    pub(crate) unit_mask_b: u64,
+    pub(crate) pstate_cap: u64,
+}
+
+impl Raw {
+    /// No parameter given.
+    #[cfg(test)]
+    pub(crate) const NONE: Raw = Raw {
+        start: UNSET,
+        initdata_version: UNSET,
+        fender: UNSET,
+        clkgen: UNSET,
+        sgx_setup: UNSET,
+        unit_mask_a: UNSET,
+        unit_mask_b: UNSET,
+        pstate_cap: UNSET,
+    };
+
+    /// Whether any parameter other than `asahi.t8122_start` was given.
+    pub(crate) fn values_given(&self) -> bool {
+        [
+            self.initdata_version,
+            self.fender,
+            self.clkgen,
+            self.sgx_setup,
+            self.unit_mask_a,
+            self.unit_mask_b,
+            self.pstate_cap,
+        ]
+        .iter()
+        .any(|v| *v != UNSET)
+    }
+}
+
+/// What `asahi.t8122_start` asks for.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) enum Start {
+    /// Not given, or 0.
+    Off,
+    /// 1.
+    On,
+    /// Anything else.
+    Invalid,
+}
+
+pub(crate) fn start(raw: &Raw) -> Start {
+    match raw.start {
+        UNSET | 0 => Start::Off,
+        1 => Start::On,
+        _ => Start::Invalid,
+    }
+}
+
+/// The shipped board policy changes only an omitted start parameter. Explicit recovery-off,
+/// explicit experimental start, and malformed values retain their original meanings.
+/// Hardware resources and firmware identity are admitted separately by the caller.
+pub(crate) fn start_for_board(raw: &Raw, admitted_board: bool) -> Start {
+    if raw.start == UNSET && admitted_board {
+        Start::On
+    } else {
+        start(raw)
+    }
+}
+
+/// The GPU clock-generator IO mapping.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) enum ClockGen {
+    /// SGX + the offset, mapped read-only for the firmware.
+    At(u64),
+    /// No mapping: HwDataB slot 29 stays zero.
+    Absent,
+}
+
+/// The values of one boot.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) struct Values {
+    pub(crate) initdata_version: u64,
+    pub(crate) fender: u32,
+    pub(crate) clock_gen: ClockGen,
+    pub(crate) sgx_setup: Option<(usize, u32)>,
+    pub(crate) unit_mask_a: u64,
+    pub(crate) unit_mask_b: u32,
+    pub(crate) pstate_cap: u32,
+}
+
+/// A parameter given with a value it does not accept: its name and what it accepts.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) struct Refusal {
+    pub(crate) name: &'static str,
+    pub(crate) accepts: &'static str,
+}
+
+/// A parameter's value, its default when not given; None when invalid.
+fn given(param: u64, default: u64) -> Option<u64> {
+    match param {
+        UNSET => Some(default),
+        INVALID => None,
+        v => Some(v),
+    }
+}
+
+/// Resolve the parameters. The first parameter with a value it does not accept is the refusal.
+pub(crate) fn resolve(raw: &Raw) -> Result<Values, Refusal> {
+    let refuse = |name, accepts| Refusal { name, accepts };
+    let initdata_version = given(raw.initdata_version, crate::m3::m3_firmware::G15_V14_8_3_INITDATA)
+        .filter(|v| *v != 0)
+        .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?;
+    let fender = match given(raw.fender, FENDER_RULE as u64) {
+        Some(v) if v == FENDER_RULE as u64 || v == FENDER_ADT as u64 => v as u32,
+        _ => return Err(refuse("t8122_fender", "0x104000 or rule, 0x12c000 or adt")),
+    };
+    let clock_gen = match given(raw.clkgen, CLOCK_GEN_E1C) {
+        Some(0) => ClockGen::Absent,
+        Some(v) if v == CLOCK_GEN_E1C || v == CLOCK_GEN_E5C => ClockGen::At(v),
+        _ => return Err(refuse("t8122_clkgen", "e1c, e5c or none")),
+    };
+    let sgx_setup = match given(raw.sgx_setup, 1) {
+        Some(0) => None,
+        Some(1) => Some(T6030_SGX_SETUP),
+        _ => return Err(refuse("t8122_sgx_setup", "none or t6030")),
+    };
+    let unit_mask_a = given(raw.unit_mask_a, UNIT_MASK_A)
+        .filter(|v| *v != 0 && *v & !UNIT_MASK_A_LIMIT == 0)
+        .ok_or(refuse("t8122_unit_mask_a", "nonzero, no bits outside 0x700000003"))?;
+    let unit_mask_b = given(raw.unit_mask_b, UNIT_MASK_B as u64)
+        .filter(|v| *v != 0 && *v & !UNIT_MASK_B_LIMIT == 0)
+        .ok_or(refuse("t8122_unit_mask_b", "nonzero, no bits outside 0x7"))? as u32;
+    let pstate_cap = given(raw.pstate_cap, PSTATE_CAP as u64)
+        .filter(|v| (1..=u64::from(PSTATE_CAP_MAX)).contains(v))
+        .ok_or(refuse("t8122_pstate_cap", "1 to 5"))? as u32;
+    Ok(Values {
+        initdata_version,
+        fender,
+        clock_gen,
+        sgx_setup,
+        unit_mask_a,
+        unit_mask_b,
+        pstate_cap,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn with(edit: impl Fn(&mut Raw)) -> Raw {
+        let mut raw = Raw { start: 1, ..Raw::NONE };
+        edit(&mut raw);
+        raw
+    }
+
+    #[test]
+    fn nothing_given_is_off_and_gives_no_values() {
+        assert_eq!(start(&Raw::NONE), Start::Off);
+        assert!(!Raw::NONE.values_given());
+        assert_eq!(start(&with(|r| r.start = 0)), Start::Off);
+        assert_eq!(start(&with(|_| {})), Start::On);
+        assert_eq!(start(&with(|r| r.start = 2)), Start::Invalid);
+        assert_eq!(start(&with(|r| r.start = INVALID)), Start::Invalid);
+        assert!(!with(|_| {}).values_given());
+        assert!(with(|r| r.pstate_cap = 3).values_given());
+        assert!(with(|r| r.fender = INVALID).values_given());
+    }
+
+    #[test]
+    fn automatic_start_is_board_scoped_and_explicit_values_win() {
+        assert_eq!(start_for_board(&Raw::NONE, true), Start::On);
+        assert_eq!(start_for_board(&Raw::NONE, false), Start::Off);
+        for admitted_board in [false, true] {
+            assert_eq!(start_for_board(&with(|r| r.start = 0), admitted_board), Start::Off);
+            assert_eq!(start_for_board(&with(|_| {}), admitted_board), Start::On);
+            assert_eq!(start_for_board(&with(|r| r.start = 2), admitted_board), Start::Invalid);
+            assert_eq!(start_for_board(&with(|r| r.start = INVALID), admitted_board), Start::Invalid);
+            assert_eq!(start_for_board(&with(|r| r.start = parse_number("bogus")), admitted_board), Start::Invalid);
+        }
+    }
+
+    #[test]
+    fn defaults_are_the_plan_baseline() {
+        let v = resolve(&with(|_| {})).unwrap();
+        assert_eq!(v.initdata_version, 0x0c08_e21e_8380_0490);
+        assert_eq!(v.fender, 0x10_4000);
+        assert_eq!(v.clock_gen, ClockGen::At(0xe1_c000));
+        assert_eq!(v.sgx_setup, Some((0xd1_4000, 0x7_0001)));
+        assert_eq!((v.unit_mask_a, v.unit_mask_b), (0x7_0000_0003, 7));
+        assert_eq!(v.pstate_cap, 2);
+    }
+
+    #[test]
+    fn every_alternative_resolves() {
+        let v = resolve(&with(|r| {
+            r.initdata_version = parse_number("0x123456789abcdef0");
+            r.fender = parse_fender("adt");
+            r.clkgen = parse_clkgen("e5c");
+            r.sgx_setup = parse_sgx_setup("none");
+            r.unit_mask_a = parse_number("1");
+            r.unit_mask_b = parse_number("0x1");
+            r.pstate_cap = parse_number("5");
+        }))
+        .unwrap();
+        assert_eq!(v.initdata_version, 0x1234_5678_9abc_def0);
+        assert_eq!(v.fender, 0x12_c000);
+        assert_eq!(v.clock_gen, ClockGen::At(0xe5_c000));
+        assert_eq!(v.sgx_setup, None);
+        assert_eq!((v.unit_mask_a, v.unit_mask_b, v.pstate_cap), (1, 1, 5));
+        let v = resolve(&with(|r| r.clkgen = parse_clkgen("none"))).unwrap();
+        assert_eq!(v.clock_gen, ClockGen::Absent);
+        // Numbers and names give the same values.
+        assert_eq!(parse_fender("0x104000"), parse_fender("rule"));
+        assert_eq!(parse_fender(" 0x12C000 "), parse_fender("adt"));
+        assert_eq!(parse_clkgen("0xe5c000"), parse_clkgen("e5c"));
+        assert_eq!(parse_clkgen("0xe1c000"), parse_clkgen("e1c"));
+        assert_eq!(parse_clkgen("0"), parse_clkgen("none"));
+        assert_eq!(parse_sgx_setup("1"), parse_sgx_setup("t6030"));
+    }
+
+    #[test]
+    fn unaccepted_values_refuse_and_name_the_parameter() {
+        let cases: [(fn(&mut Raw), &str); 15] = [
+            (|r| r.initdata_version = parse_number("g15s"), "t8122_initdata_version"),
+            (|r| r.initdata_version = 0, "t8122_initdata_version"),
+            (|r| r.fender = parse_fender("0x144000"), "t8122_fender"),
+            (|r| r.fender = parse_fender("big"), "t8122_fender"),
+            (|r| r.clkgen = parse_clkgen("0xe00000"), "t8122_clkgen"),
+            (|r| r.clkgen = parse_clkgen("e5"), "t8122_clkgen"),
+            (|r| r.sgx_setup = parse_sgx_setup("2"), "t8122_sgx_setup"),
+            (|r| r.unit_mask_a = parse_number("0x700000007"), "t8122_unit_mask_a"),
+            (|r| r.unit_mask_a = 0, "t8122_unit_mask_a"),
+            (|r| r.unit_mask_b = parse_number("0xf"), "t8122_unit_mask_b"),
+            (|r| r.unit_mask_b = 0, "t8122_unit_mask_b"),
+            (|r| r.pstate_cap = 0, "t8122_pstate_cap"),
+            (|r| r.pstate_cap = 6, "t8122_pstate_cap"),
+            (|r| r.pstate_cap = 16, "t8122_pstate_cap"),
+            (|r| r.pstate_cap = parse_number("-1"), "t8122_pstate_cap"),
+        ];
+        for (edit, name) in cases {
+            assert_eq!(resolve(&with(edit)).unwrap_err().name, name);
+        }
+    }
+
+    #[test]
+    fn pstate_cap_accepts_one_to_five_only() {
+        for cap in 1..=PSTATE_CAP_MAX {
+            assert_eq!(resolve(&with(|r| r.pstate_cap = u64::from(cap))).unwrap().pstate_cap, cap);
+        }
+        for cap in [0u64, 6, 8, 15, 16] {
+            assert!(resolve(&with(|r| r.pstate_cap = cap)).is_err(), "cap {cap}");
+        }
+        assert_eq!(PSTATE_CAP_MAX, 5);
+    }
+
+    #[test]
+    fn a_leading_sign_is_rejected() {
+        for text in ["+1", "-1", "0x+1", "0x-1", "+0x10"] {
+            assert_eq!(parse_number(text), INVALID, "{text:?}");
+        }
+        assert!(resolve(&with(|r| r.pstate_cap = parse_number("+3"))).is_err());
+    }
+
+    #[test]
+    fn reserved_numbers_and_garbage_parse_as_invalid() {
+        for text in ["", "0x", "x1", "18446744073709551615", "18446744073709551614", "0x1g"] {
+            assert_eq!(parse_number(text), INVALID, "{text:?}");
+        }
+        assert_eq!(parse_number("18446744073709551613"), INVALID - 1);
+        assert_eq!(parse_number(" 0X10 "), 16);
+    }
+
+    #[test]
+    fn default_masks_are_within_the_limits() {
+        assert_eq!(UNIT_MASK_A & !UNIT_MASK_A_LIMIT, 0);
+        assert_eq!(u64::from(UNIT_MASK_B) & !UNIT_MASK_B_LIMIT, 0);
+        assert_eq!(UNIT_MASK_A_LIMIT, 0x7_0000_0003);
+        assert_eq!(UNIT_MASK_B_LIMIT, 0x7);
+    }
+}
diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs
new file mode 100644
index 0000000000000..737034dd6f8c1
--- /dev/null
+++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs
@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! T8122 (M3, G15G) board-default startup and bounded experiment values.
+//!
+//! The T8122 table (`m3_soc::T8122`) is complete. An omitted start parameter starts
+//! automatically only on its listed Air boards; explicit `asahi.t8122_start=0`
+//! remains the recovery opt-out. Resource and known-firmware admission still run.
+//! The same table values can be changed for a controlled experiment by
+//! rebooting with a parameter:
+//!
+//! | Parameter | Default | Others |
+//! |---|---|---|
+//! | `asahi.t8122_initdata_version` | `0x0c08e21e83800490`, the G15 14.8.3 version | any |
+//! | `asahi.t8122_fender` | `0x104000` (`rule`) | `0x12c000` (`adt`) |
+//! | `asahi.t8122_clkgen` | `e1c`: SGX+0xe1c000, read-only | `e5c`: SGX+0xe5c000, read-only; `none` |
+//! | `asahi.t8122_sgx_setup` | `t6030`: SGX+0xd14000 = 0x70001 | `none` |
+//! | `asahi.t8122_unit_mask_a` | `0x700000003` | nonzero, within `0x700000003` |
+//! | `asahi.t8122_unit_mask_b` | `0x7` | nonzero, within `0x7` |
+//! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 |
+//!
+//! The defaults are the T8122 table's values. A parameter given with a value it does not accept
+//! refuses the start. Unlisted boards remain refused by the existing backend admission.
+//! Nothing here applies to any other SoC.
+
+use core::mem::offset_of;
+
+use kernel::{device, prelude::*};
+
+use crate::{
+    m3::g15_raw::raw,
+    m3::initdata::G15RuntimeHwDataB,
+    m3::m3_adt_config::T8122_IO_MAPPINGS,
+    m3::m3_init_storage::{self as storage, IoMap},
+    m3::m3_soc::{IoMapping, Soc, T8122_HWDATA_B},
+    pgtable::{prot, Prot},
+    m3::t8122_knobs::{
+        self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT,
+        FENDER_RULE,
+    },
+};
+
+/// The GPU register window (SGX) of T8122.
+const SGX: u64 = 0x2_9000_0000;
+/// Index of the Fender window and of the GPU clock generator in [`T8122_IO_MAPPINGS`].
+const FENDER: usize = 0;
+const CLOCK_GEN: usize = 11;
+/// The HwDataB slot of the GPU clock generator.
+const CLOCK_GEN_SLOT: usize = 29;
+
+
+/// The firmware IO mappings with the Fender window `fender` and the clock generator at
+/// SGX + `clock_gen`.
+const fn mappings(fender: u32, clock_gen: u64) -> [IoMapping; 12] {
+    let mut m = T8122_IO_MAPPINGS;
+    m[FENDER].2 = fender;
+    m[FENDER].3 = fender;
+    m[CLOCK_GEN].1 = SGX + clock_gen;
+    m
+}
+
+/// The runtime IO maps, laid out once for the larger Fender window so that no slot's firmware
+/// VA depends on the parameters.
+const LAYOUT: [IoMap; 12] =
+    storage::pack_iomaps(&mappings(FENDER_ADT, CLOCK_GEN_E1C), storage::IOMAP_BASE);
+
+const fn iomaps(fender: u32, clock_gen: u64) -> [IoMap; 12] {
+    let mut m = LAYOUT;
+    m[FENDER].size = fender as usize;
+    m[CLOCK_GEN].physical = SGX + clock_gen;
+    m
+}
+
+/// Every Fender window and clock-generator address, in [`Experiment::variant`] order.
+static IO_MAPPINGS: [[IoMapping; 12]; 4] = [
+    mappings(FENDER_RULE, CLOCK_GEN_E5C),
+    mappings(FENDER_RULE, CLOCK_GEN_E1C),
+    mappings(FENDER_ADT, CLOCK_GEN_E5C),
+    mappings(FENDER_ADT, CLOCK_GEN_E1C),
+];
+static IOMAPS: [[IoMap; 12]; 4] = [
+    iomaps(FENDER_RULE, CLOCK_GEN_E5C),
+    iomaps(FENDER_RULE, CLOCK_GEN_E1C),
+    iomaps(FENDER_ADT, CLOCK_GEN_E5C),
+    iomaps(FENDER_ADT, CLOCK_GEN_E1C),
+];
+
+/// Whether each IO map holds its IO mapping: same slot, the mapping's 16 KiB-aligned physical
+/// base, its subpage offset, and room for its total size.
+const fn maps_cover(mappings: &[IoMapping; 12], maps: &[IoMap; 12]) -> bool {
+    let mut i = 0;
+    while i < 12 {
+        let (slot, phys, total, _, _) = mappings[i];
+        let io = maps[i];
+        if io.slot != slot
+            || io.physical + io.offset as u64 != phys
+            || io.offset + total as usize > io.size
+            || io.size % 0x4000 != 0
+        {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+const _: () = {
+    // The default variant's IO mappings are the T8122 table's.
+    let defaults = mappings(FENDER_RULE, CLOCK_GEN_E1C);
+    let mut j = 0;
+    while j < 12 {
+        let (a, b) = (defaults[j], T8122_IO_MAPPINGS[j]);
+        assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2 && a.3 == b.3 && a.4 == b.4);
+        j += 1;
+    }
+    // The T8122 table: the Fender window at the rule size, the clock generator last, at
+    // SGX+0xe1c000, 16 KiB, read-only.
+    assert!(T8122_IO_MAPPINGS[FENDER].0 == 0 && T8122_IO_MAPPINGS[FENDER].1 == SGX + 0xd0_0000);
+    assert!(T8122_IO_MAPPINGS[FENDER].2 == FENDER_RULE);
+    assert!(T8122_IO_MAPPINGS[CLOCK_GEN].0 == CLOCK_GEN_SLOT);
+    assert!(T8122_IO_MAPPINGS[CLOCK_GEN].1 == SGX + CLOCK_GEN_E1C);
+    assert!(T8122_IO_MAPPINGS[CLOCK_GEN].2 == 0x4000 && !T8122_IO_MAPPINGS[CLOCK_GEN].4);
+    assert!(CLOCK_GEN + 1 == T8122_IO_MAPPINGS.len());
+    // Both windows are whole pages and hold the Fender scratch area (+0x60000..+0x80000).
+    assert!(FENDER_RULE % 0x4000 == 0 && FENDER_ADT % 0x4000 == 0 && FENDER_RULE >= 0x8_0000);
+    assert!(FENDER_RULE < FENDER_ADT);
+    // Both clock-generator blocks are whole pages inside the first 16 MiB of SGX.
+    assert!(CLOCK_GEN_E5C % 0x4000 == 0 && CLOCK_GEN_E1C % 0x4000 == 0);
+    assert!(CLOCK_GEN_E5C + 0x4000 <= 0x100_0000 && CLOCK_GEN_E1C + 0x4000 <= 0x100_0000);
+    let mut i = 0;
+    while i < 4 {
+        assert!(maps_cover(&IO_MAPPINGS[i], &IOMAPS[i]));
+        i += 1;
+    }
+    // The default unit masks are the T8122 table's, and the limits allow them.
+    assert!(T8122_HWDATA_B.unit_mask_a == knobs::UNIT_MASK_A);
+    assert!(T8122_HWDATA_B.unit_mask_b == knobs::UNIT_MASK_B);
+    assert!(knobs::UNIT_MASK_A & !knobs::UNIT_MASK_A_LIMIT == 0);
+    assert!(knobs::UNIT_MASK_B as u64 & !knobs::UNIT_MASK_B_LIMIT == 0);
+    assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_a) == 0x17c0);
+    assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8);
+    // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the
+    // opt-in itself; if another is added, this fails the build until the start accounts for it.
+    assert!(crate::m3::m3_soc::T8122.unported.len() == 1);
+};
+
+/// The values of one armed boot.
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct Experiment(Values);
+
+/// Whether `soc` is the T8122 table.
+pub(crate) fn is_t8122(soc: &Soc) -> bool {
+    core::ptr::eq(soc, &crate::m3::m3_soc::T8122)
+}
+
+impl Experiment {
+    /// Index of this boot's IO-mapping table in [`IO_MAPPINGS`] and [`IOMAPS`].
+    fn variant(&self) -> usize {
+        let adt = usize::from(self.0.fender == FENDER_ADT);
+        let e1c = usize::from(self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C));
+        2 * adt + e1c
+    }
+
+    /// How many IO mappings this boot gives the firmware: all, or all but the clock generator.
+    fn mapping_count(&self) -> usize {
+        match self.0.clock_gen {
+            ClockGen::At(_) => CLOCK_GEN + 1,
+            ClockGen::Absent => CLOCK_GEN,
+        }
+    }
+
+    /// The InitData version given to the firmware (`asahi.t8122_initdata_version`).
+    pub(crate) fn initdata_version(&self) -> u64 {
+        self.0.initdata_version
+    }
+
+    /// The firmware IO mappings of the InitData.
+    pub(crate) fn io_mappings(&self) -> &'static [IoMapping] {
+        &IO_MAPPINGS[self.variant()][..self.mapping_count()]
+    }
+
+    /// The runtime's mapping of each of them. With the default Fender window and clock generator
+    /// they are the T8122 table's own IO maps.
+    pub(crate) fn iomaps(&self) -> &'static [IoMap] {
+        if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) {
+            return &crate::m3::m3_adt_config::T8122_IOMAPS;
+        }
+        &IOMAPS[self.variant()][..self.mapping_count()]
+    }
+
+    /// The register block the IO mapping of `slot` must lie in, given the HwConfig's block
+    /// `base` for it: the selected clock generator for slot 29.
+    pub(crate) fn io_block(&self, slot: usize, base: u64) -> u64 {
+        match self.0.clock_gen {
+            ClockGen::At(offset) if slot == CLOCK_GEN_SLOT => SGX + offset,
+            _ => base,
+        }
+    }
+
+    /// The HwDataB slots the firmware maps read-only, as a bit mask: the clock generator's.
+    pub(crate) fn read_only_slots(&self) -> u32 {
+        match self.0.clock_gen {
+            ClockGen::At(_) => 1 << CLOCK_GEN_SLOT,
+            ClockGen::Absent => 0,
+        }
+    }
+
+    /// The runtime HwDataB words: the T8122 table's. The unit masks are written over them
+    /// ([`Self::unit_masks`]).
+    pub(crate) fn hwdata_b(&self) -> &'static G15RuntimeHwDataB {
+        &T8122_HWDATA_B
+    }
+
+    /// The HwDataB unit masks A (+0x17c0) and B (+0x17c8).
+    pub(crate) fn unit_masks(&self) -> (u64, u32) {
+        (self.0.unit_mask_a, self.0.unit_mask_b)
+    }
+
+    /// The SGX write (offset, value) made before the firmware starts, if any.
+    pub(crate) fn sgx_setup(&self) -> Option<(usize, u32)> {
+        self.0.sgx_setup
+    }
+
+    /// The highest performance state the firmware may use (`asahi.t8122_pstate_cap`).
+    pub(crate) fn pstate_cap(&self) -> u32 {
+        self.0.pstate_cap
+    }
+}
+
+/// Firmware page protection of the IO map of `slot`, with the read-only slots `read_only`.
+pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot {
+    if slot < 32 && read_only & (1 << slot) != 0 {
+        prot::PROT_FW_MMIO_RO
+    } else {
+        prot::PROT_FW_MMIO_RW
+    }
+}
+
+/// Decide, after resource admission, whether `soc` starts with the experiment values.
+///
+/// Any SoC but T8122: Ok(None). On its listed Air boards, an omitted parameter
+/// selects the table values; explicit 0 disables startup, and invalid values fail
+/// closed. Existing backend admission still refuses an unlisted board, even
+/// with an explicit start request.
+pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result<Option<Experiment>> {
+    if !is_t8122(soc) {
+        return Ok(None);
+    }
+    let raw = crate::m3::m3_params::t8122_params();
+    match knobs::start_for_board(&raw, crate::m3::m3_board::runtime_validated_board(soc)) {
+        Start::On => {}
+        Start::Off => {
+            if raw.values_given() {
+                dev_info!(
+                    dev,
+                    "M3 G15G start: disabled by asahi.t8122_start=0; other asahi.t8122_* values ignored\n"
+                );
+            }
+            return Ok(None);
+        }
+        Start::Invalid => {
+            dev_err!(
+                dev,
+                "M3 G15G start: refused: asahi.t8122_start has a value it does not accept (0 or 1); GPU startup disabled\n"
+            );
+            return Err(ENODEV);
+        }
+    }
+    // Admission has required the boot loader's handoff: its firmware segments and reserved
+    // regions, and on this SoC its power configuration.
+    if !soc.power_from_boot_loader || soc.firmware.is_none() || soc.hwcfg.is_none() {
+        dev_err!(dev, "M3 G15G start: refused: the T8122 table is not the one this experiment expects\n");
+        return Err(ENODEV);
+    }
+    let v = knobs::resolve(&raw).map_err(|refusal| {
+        dev_err!(
+            dev,
+            "M3 G15G start: refused: asahi.{} has a value it does not accept ({}); GPU startup disabled\n",
+            refusal.name,
+            refusal.accepts
+        );
+        ENODEV
+    })?;
+    let e = Experiment(v);
+    dev_warn!(
+        dev,
+        "M3 G15G start: armed (board default or asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n",
+        v.initdata_version,
+        v.fender,
+        clkgen_label(v.clock_gen, v.fender),
+        match v.sgx_setup {
+            Some(_) => "t6030(sgx+0xd14000=0x70001)",
+            None => "none",
+        },
+        v.unit_mask_a,
+        v.unit_mask_b,
+        v.pstate_cap
+    );
+    dev_warn!(
+        dev,
+        "M3 G15G start: the T8122 table's InitData words, HwData placement and register lists; {} IO mappings at firmware VA {:#x}\n",
+        e.mapping_count(),
+        storage::IOMAP_BASE
+    );
+    // J613/J615 have no SoC die temperature zone yet, so a cap above the no-feedback limit runs
+    // the fanless GPU without any temperature feedback (`m3_thermal` holds the cap but gets no
+    // reading). The runtime cap and the firmware ceiling stay equal, because the cap is at most
+    // the no-feedback limit.
+    if v.pstate_cap > knobs::PSTATE_CAP {
+        dev_warn!(
+            dev,
+            "M3 G15G start: pstate_cap={} is above {}: no SoC die temperature zone on this board, so states up to {} run with no temperature feedback\n",
+            v.pstate_cap,
+            knobs::PSTATE_CAP,
+            v.pstate_cap
+        );
+    }
+    Ok(Some(e))
+}
+
+/// The armed line's clock-generator field. It says `ro` for the firmware's own slot-29 mapping,
+/// and adds that the Fender window reaches the block read-write when the chosen window covers it
+/// (SGX+0xe1c000 lies under the 0x12c000 window but not the 0x104000 one; SGX+0xe5c000 under
+/// neither).
+fn clkgen_label(clock_gen: ClockGen, fender: u32) -> &'static str {
+    let fender_end = 0xd0_0000u64 + u64::from(fender);
+    let reaches = |off: u64| off + 0x4000 <= fender_end;
+    match clock_gen {
+        ClockGen::At(CLOCK_GEN_E1C) if reaches(CLOCK_GEN_E1C) => {
+            "e1c(sgx+0xe1c000,ro; also rw in the fender window)"
+        }
+        ClockGen::At(CLOCK_GEN_E1C) => "e1c(sgx+0xe1c000,ro)",
+        ClockGen::At(_) if reaches(CLOCK_GEN_E5C) => {
+            "e5c(sgx+0xe5c000,ro; also rw in the fender window)"
+        }
+        ClockGen::At(_) => "e5c(sgx+0xe5c000,ro)",
+        ClockGen::Absent if reaches(CLOCK_GEN_E1C) => {
+            "none (but the fender window reaches sgx+0xe1c000 rw)"
+        }
+        ClockGen::Absent => "none",
+    }
+}
+
+/// Log, in an armed start, that the probe stopped at `stage` before the firmware started.
+pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, stage: &str, error: Error) {
+    if experiment.is_some() {
+        dev_err!(
+            dev,
+            "M3 G15G start: refused at {} ({:?}); the firmware was not started, see the lines above\n",
+            stage,
+            error
+        );
+    }
+}
+
+/// Log that resource admission refused a T8122 start selected by board default or flag.
+pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) {
+    if is_t8122(soc) && knobs::start_for_board(
+        &crate::m3::m3_params::t8122_params(),
+        crate::m3::m3_board::runtime_validated_board(soc),
+    ) == Start::On {
+        dev_err!(
+            dev,
+            "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n"
+        );
+    }
+}
+
+// One `M3 G15G verdict:` line per outcome of an armed start, so that a tester can tell them
+// apart from the log alone. They run on T8122 only (callers check `is_t8122`); nothing here
+// changes what the runtime does.
+
+/// The GPU coprocessor's start or the driver's InitData upload failed (`m3_runtime::Runtime::new`).
+/// `started`: the coprocessor ran and offered its endpoints.
+pub(crate) fn prepare_verdict(dev: &device::Device, soc: &Soc, started: bool, error: Error) {
+    if !is_t8122(soc) {
+        return;
+    }
+    if started {
+        dev_err!(
+            dev,
+            "M3 G15G verdict: driver-refused ({:?}): the GPU firmware is up, but the driver did not publish the InitData (UAT, upload checks or thermal setup, see above)\n",
+            error
+        );
+    } else {
+        dev_err!(
+            dev,
+            "M3 G15G verdict: firmware-boot-failed ({:?}): the GPU coprocessor did not start, or its RTKit did not offer endpoints 0x20/0x21; no InitData was published\n",
+            error
+        );
+    }
+}
+
+/// The step the firmware boot sequence reached (`m3_runtime::Runtime::boot_inner`). The verdict
+/// uses it to name a failure as a publish, a device-control or an InitData-acknowledgement
+/// problem, rather than lumping them all under "initdata-rejected".
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub(crate) enum BootStep {
+    /// About to send, or sending, the InitData publish message (endpoint 0x20).
+    Publish,
+    /// A device-control message (0x13 or 9) before the firmware signalled ready.
+    DeviceControl,
+    /// Waiting (up to 2 s) for the firmware to write its ready words.
+    AwaitReady,
+    /// The firmware signalled ready; a control or check after that.
+    PostReady,
+}
+
+/// The firmware's answer to the published InitData (`m3_runtime::Runtime::boot`). `accepted`:
+/// the firmware wrote its ready words. `step`: the furthest step the boot reached.
+pub(crate) fn boot_verdict(
+    dev: &device::Device,
+    version: u64,
+    step: BootStep,
+    accepted: bool,
+    crashed: bool,
+    result: Result,
+) {
+    let Err(e) = result else {
+        dev_info!(
+            dev,
+            "M3 G15G verdict: firmware-running: the firmware accepted the InitData (version {:#x}) and the device controls; no job has run yet\n",
+            version
+        );
+        return;
+    };
+    // The firmware signalled ready, then a later control or check failed: the InitData was
+    // accepted, so this is not a rejection.
+    if accepted || step == BootStep::PostReady {
+        if crashed {
+            dev_err!(
+                dev,
+                "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}) then crashed during a post-boot control or check; see the crash lines above\n",
+                e,
+                version
+            );
+        } else {
+            dev_err!(
+                dev,
+                "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}), then a post-boot control or check failed (see above)\n",
+                e,
+                version
+            );
+        }
+        return;
+    }
+    match step {
+        BootStep::Publish => dev_err!(
+            dev,
+            "M3 G15G verdict: publish-failed ({:?}): the InitData publish message (endpoint 0x20) could not be sent; the firmware was not told about the InitData (version {:#x})\n",
+            e,
+            version
+        ),
+        BootStep::DeviceControl => dev_err!(
+            dev,
+            "M3 G15G verdict: device-control-failed ({:?}): a device-control queue, mailbox send or acknowledgement failed after the InitData (version {:#x}) was published; see the device-control diagnostic above\n",
+            e,
+            version
+        ),
+        // AwaitReady (and PostReady is handled above).
+        _ if crashed => dev_err!(
+            dev,
+            "M3 G15G verdict: initdata-rejected ({:?}): the firmware crashed after the InitData (version {:#x}) was published; see the crash lines above\n",
+            e,
+            version
+        ),
+        _ => dev_err!(
+            dev,
+            "M3 G15G verdict: initdata-rejected ({:?}): the firmware did not acknowledge the InitData (version {:#x}) within 2 s (its ready words stayed unset); see M3 firmware readiness above\n",
+            e,
+            version
+        ),
+    }
+}
+
+/// A GPU coprocessor setup step failed before the firmware could be started
+/// (`m3_runtime::Runtime::new`): the RTKit state, the mailbox transport or the DRM device.
+pub(crate) fn coproc_setup_refused(dev: &device::Device, error: Error) {
+    dev_err!(
+        dev,
+        "M3 G15G start: refused at the GPU coprocessor setup (RTKit/mailbox) ({:?}); the firmware was not started\n",
+        error
+    );
+}
+
+/// The firmware ran a job above the performance-state cap (`m3_runtime`, the after-job pstate
+/// check). The `M3: firmware ... stays above the cap ...` line precedes this.
+pub(crate) fn cap_violated_verdict(dev: &device::Device, error: Error) {
+    dev_err!(
+        dev,
+        "M3 G15G verdict: cap-violated ({:?}): after a job the firmware reported a performance state above the cap; the GPU was marked failed (see the 'stays above the cap' line above)\n",
+        error
+    );
+}
+
+/// A job-path step failed outside the completion wait (`m3_runtime::Runtime::execute`): a submit,
+/// a doorbell, or a post-completion read. No dispatch outcome was determined.
+pub(crate) fn job_setup_failed_verdict(dev: &device::Device, error: Error) {
+    dev_err!(
+        dev,
+        "M3 G15G verdict: job-failed-before-wait ({:?}): a submit, doorbell or diagnostic step failed, so no dispatch outcome was determined; see the lines above\n",
+        error
+    );
+}
+
+/// A job that failed (`m3_runtime::Inner::fail`). `primary` is ETIMEDOUT when it did not retire
+/// within the runtime's per-batch bound; `start_stamp` is the job's first GPU start (CDM
+/// dispatch) timestamp, if readable; `pstate` is the GPU performance-state register, if readable.
+pub(crate) fn job_failed_verdict(
+    dev: &device::Device,
+    primary: Error,
+    start_stamp: Option<u64>,
+    pstate: Option<u32>,
+    crashed: bool,
+) {
+    if primary != ETIMEDOUT || crashed {
+        dev_err!(
+            dev,
+            "M3 G15G verdict: job-faulted ({:?}): the firmware crashed, reported an error, or the GPU reported a fault while the job ran; see the lines around this one\n",
+            primary
+        );
+        return;
+    }
+    // A nonzero start timestamp means the firmware dispatched the job on the GPU, so this was not
+    // a dispatch gap however the power state reads now (the GPU may have idled off since). The
+    // stamp and pstate print as `Some(hex)` or `None`.
+    if start_stamp.is_some_and(|start| start != 0) {
+        dev_err!(
+            dev,
+            "M3 G15G verdict: job-ran-completion-missed: the job was dispatched (GPU start timestamp {:x?}) but its completion was not seen within the per-batch bound (pstate register {:x?})\n",
+            start_stamp,
+            pstate
+        );
+        return;
+    }
+    // The GPU start timestamp is 0 (or unreadable): the job was not dispatched.
+    match pstate {
+        Some(p) if p & 0xf == 0 => dev_err!(
+            dev,
+            "M3 G15G verdict: job-accepted-never-dispatched: the job was not dispatched (GPU start timestamp {:x?}) and the GPU reads powered down (pstate register {:#x})\n",
+            start_stamp,
+            p
+        ),
+        Some(p) => dev_err!(
+            dev,
+            "M3 G15G verdict: job-timed-out-powered: the job did not retire with the GPU powered (pstate register {:#x}, GPU start timestamp {:x?}); see the engine snapshot below\n",
+            p,
+            start_stamp
+        ),
+        None => dev_err!(
+            dev,
+            "M3 G15G verdict: job-timed-out: the job did not retire (pstate register and GPU start timestamp both unreadable)\n"
+        ),
+    }
+}
+
+/// The first job of a kind (0 render, 1 compute) retired (`m3_runtime::Runtime::execute`):
+/// `span` is its first start and last end GPU timestamp (24 MHz ticks), if readable.
+pub(crate) fn job_completed_verdict(dev: &device::Device, kind: usize, gpu_ns: u64, span: Option<[u64; 2]>) {
+    let what = if kind == 0 { "render" } else { "compute" };
+    match span {
+        Some([start, end]) if start != 0 && end != 0 => dev_info!(
+            dev,
+            "M3 G15G verdict: job-completed: the first {} job finished, GPU timestamps {:#x}..{:#x}, {} ns of GPU time\n",
+            what,
+            start,
+            end,
+            gpu_ns
+        ),
+        Some([start, end]) => dev_err!(
+            dev,
+            "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are {:#x}..{:#x}\n",
+            what,
+            start,
+            end
+        ),
+        None => dev_err!(
+            dev,
+            "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are unreadable\n",
+            what
+        ),
+    }
+}
diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs
index 4cc9bf4223faf..f2d078c358264 100644
--- a/drivers/gpu/drm/asahi/mmu.rs
+++ b/drivers/gpu/drm/asahi/mmu.rs
@@ -116,6 +116,8 @@ mod lifetime;
 #[cfg(CONFIG_DRM_ASAHI_M5)]
 mod m5_context;
 mod ppl;
+#[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+mod m3;
 mod residency;
 mod shared;
 #[cfg(CONFIG_DRM_ASAHI_G18)]
@@ -1270,8 +1272,7 @@ impl VmBind {
         self.1
     }
 
-    /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound.
-    #[cfg(CONFIG_DRM_ASAHI_G16)]
+    /// Stable firmware wire incarnation for this active root lease; zero is unbound.
     pub(crate) fn generation(&self) -> u8 {
         self.0.binding.lock().generation
     }
@@ -2897,7 +2898,6 @@ impl Vm {
     }
 
     /// Spend the binding generation only for an incarnation exposed to firmware.
-    #[cfg(CONFIG_DRM_ASAHI_G16)]
     pub(crate) fn mark_binding_published(&self) -> Result {
         let uat = self.gpuvm.lock_inner().uat_inner.clone();
         let binding = self.binding.lock();
@@ -3334,7 +3334,7 @@ impl Uat {
     /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also
     /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access;
     /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper.
-    #[cfg(CONFIG_DRM_ASAHI_G16)]
+    #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_M3_T8122))]
     pub(crate) fn new_adopted_with_user_upper(
         dev: &driver::AsahiDrmDevice,
         geometry: UatGeometry,
diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs
new file mode 100644
index 0000000000000..e9dc7184d6582
--- /dev/null
+++ b/drivers/gpu/drm/asahi/mmu/m3.rs
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! The VM and binding helpers of the M3 T8122 runtime (`crate::m3`), which 7.1.12's `mmu.rs`
+//! provided for it.
+
+use super::*;
+
+impl Vm {
+    /// Whether `other` is the same DRM VM.
+    pub(crate) fn same(&self, other: &Vm) -> bool {
+        self.id == other.id
+    }
+
+    /// Copy the GPU-readable bytes this VM maps at `iova`, through the point of coherency
+    /// ([`Vm::read_user`]). M3 compute validates the client control stream before publishing it.
+    pub(crate) fn read_bytes(&self, iova: u64, out: &mut [u8]) -> Result {
+        if out.is_empty() {
+            return Err(EINVAL);
+        }
+        self.read_user(iova, out)
+    }
+
+    /// Snapshot the first `out.len()` TTBAT entries as `(ttb0, ttb1)` pairs. The TTBAT is
+    /// ordinary memory, so this reads no GPU register and is safe with the cores gated.
+    pub(crate) fn context_roots(&self, out: &mut [(u64, u64)]) -> Result {
+        let inner = lock_vm(&self.gpuvm, None, false)?;
+        let shared = inner.uat_inner.lock();
+        let ttbs = shared.ttbs();
+        for (slot, entry) in out.iter_mut().enumerate().take(UAT_NUM_CTX) {
+            *entry = (
+                ttbs[slot].ttb0.load(Ordering::Acquire),
+                ttbs[slot].ttb1.load(Ordering::Acquire),
+            );
+        }
+        Ok(())
+    }
+
+    /// Map `size` bytes of MMIO or DMA memory at `phys` into this Vm, at `iova`, or at a free
+    /// address in `range` when `iova` is `None`. Unlike [`Vm::map_io`] this logs nothing on
+    /// success: the runtime maps job views on every submission.
+    fn map_io_m3_inner(
+        &self,
+        iova: Option<u64>,
+        range: Range<u64>,
+        phys: usize,
+        size: usize,
+        prot: Prot,
+    ) -> Result<KernelMapping> {
+        if size == 0 || (iova.unwrap_or(0) as usize | phys | size) & UAT_PGMSK != 0 {
+            pr_err!(
+                "MMU: M3 KernelMapping {:#x}:{:#x} -> {:#x?} is not page-aligned\n",
+                phys,
+                size,
+                iova
+            );
+            return Err(EINVAL);
+        }
+        // SAFETY: As in `map_io`, the exec lock is only used for this mapping.
+        let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? };
+        let inner = vm_exec.unique_gpuvm.data();
+        let payload = KernelMappingInner {
+            owner: self.gpuvm.clone(),
+            uat_inner: inner.uat_inner.clone(),
+            prot,
+            bo: None,
+            _gem: None,
+            offset: 0,
+            mapped_size: size,
+        };
+        let node = match iova {
+            Some(iova) => inner.mm.reserve_node(payload, iova, size as u64, 0)?,
+            None => inner.mm.insert_node_in_range(
+                payload,
+                size as u64,
+                UAT_PGSZ as u64,
+                0,
+                range.start,
+                range.end,
+                mm::InsertMode::Best,
+            )?,
+        };
+        let iova = node.start();
+        let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?;
+        inner.page_table.prepare_map(iova..end)?;
+        let transaction_uat = inner.uat_inner.clone();
+        let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?;
+        let mutation = inner.mapping_mutation();
+        let ret = inner
+            .page_table
+            .map_pages(iova..end, phys as PhysicalAddr, prot, false);
+        let discard = if ret.is_err() {
+            inner.page_table.discard_partial_map(iova..end)
+        } else {
+            Ok(())
+        };
+        drop(handoff);
+        if ret.is_err() {
+            inner.finish_failed_map(iova, size);
+        } else {
+            inner.tlbi_contexts_naming_root(iova, size);
+        }
+        let retain = discard.is_err() && inner.uat_inner.m4_adopted;
+        if retain {
+            inner.uat_inner.failed.store(true, Ordering::Release);
+        }
+        drop(mutation);
+        drop(vm_exec);
+        if retain {
+            core::mem::forget(node);
+            return Err(ret.err().unwrap_or(EIO));
+        }
+        ret?;
+        Ok(KernelMapping(ManuallyDrop::new(node)))
+    }
+
+    /// [`Vm::map_io`] at `iova`, without the per-mapping log line.
+    pub(crate) fn map_io_m3(
+        &self,
+        iova: u64,
+        phys: usize,
+        size: usize,
+        prot: Prot,
+    ) -> Result<KernelMapping> {
+        self.map_io_m3_inner(Some(iova), 0..0, phys, size, prot)
+    }
+
+    /// [`Vm::map_io`] at a free address in `range`.
+    pub(crate) fn map_io_in_range(
+        &self,
+        range: Range<u64>,
+        phys: usize,
+        size: usize,
+        prot: Prot,
+    ) -> Result<KernelMapping> {
+        self.map_io_m3_inner(None, range, phys, size, prot)
+    }
+}
+
+impl VmBind {
+    /// The bound `Vm` itself, for diagnostics that need to walk its tables.
+    pub(crate) fn vm(&self) -> &Vm {
+        &self.0
+    }
+
+    /// Whether this retained binding belongs to the supplied DRM VM.
+    pub(crate) fn matches(&self, vm: &Vm) -> bool {
+        self.0.id == vm.id
+    }
+
+    /// Physical base of the bound VM's page-table root.
+    pub(crate) fn root(&self) -> u64 {
+        self.0.ttb()
+    }
+
+    /// The bound VM's driver-assigned id.
+    pub(crate) fn vm_id(&self) -> u64 {
+        self.0.id
+    }
+}
diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs
index ac9d7ab5032b6..1524127cd475a 100644
--- a/rust/kernel/soc/apple/rtkit.rs
+++ b/rust/kernel/soc/apple/rtkit.rs
@@ -387,6 +387,18 @@ impl<T: Operations> RtKit<T> {
         })
     }
 
+    /// Returns whether the coprocessor is running (not crashed, IOP and AP power states on).
+    pub fn is_running(&self) -> bool {
+        // SAFETY: `rtk` is valid per the type invariant. The call only reads state.
+        unsafe { bindings::apple_rtkit_is_running(self.rtk) }
+    }
+
+    /// Returns whether the coprocessor has reported a crash.
+    pub fn is_crashed(&self) -> bool {
+        // SAFETY: `rtk` is valid per the type invariant. The call only reads state.
+        unsafe { bindings::apple_rtkit_is_crashed(self.rtk) }
+    }
+
     /// Checks if an endpoint is present
     pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool {
         unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) }
diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs
index 35ff049ff0783..095a51627edfe 100644
--- a/rust/kernel/sync/completion.rs
+++ b/rust/kernel/sync/completion.rs
@@ -90,6 +90,28 @@ impl Completion {
         self.inner.get()
     }
 
+    /// Publish one completion token, including from interrupt context.
+    #[inline]
+    pub fn complete(&self) {
+        // SAFETY: The pinned completion remains initialized for this call.
+        unsafe { bindings::complete(self.as_raw()) };
+    }
+
+    /// Consume one pending token without blocking.
+    #[inline]
+    pub fn try_wait_for_completion(&self) -> bool {
+        // SAFETY: The pinned completion remains initialized for this call.
+        unsafe { bindings::try_wait_for_completion(self.as_raw()) }
+    }
+
+    /// Wait for a token for at most `timeout` jiffies; return whether one arrived.
+    #[inline]
+    pub fn wait_for_completion_timeout(&self, timeout: crate::time::Jiffies) -> bool {
+        let timeout = timeout.min(core::ffi::c_long::MAX as _);
+        // SAFETY: The completion remains live while this method waits.
+        unsafe { bindings::wait_for_completion_timeout(self.as_raw(), timeout as _) != 0 }
+    }
+
     /// Signal all tasks waiting on this completion.
     ///
     /// This method wakes up all tasks waiting on this completion; after this operation the
diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs
index a6eb52bbf4fe7..d9a0a35c04dd0 100644
--- a/tools/asahi/m3-batching-tests.rs
+++ b/tools/asahi/m3-batching-tests.rs
@@ -163,3 +163,51 @@ fn experimental_air_is_active_while_pro_retains_serial_default() {
     assert!(default_depth(true)<=SLOTS);
     assert_eq!(MAX_PACKETS,8);
 }
+
+#[test]
+fn firmware_starts_once_per_boot() {
+    let latch=StartLatch::new();
+    assert!(latch.claim());
+    // Unbind and rebind, a reload or a retry: every later start is refused.
+    for _ in 0..3 {assert!(!latch.claim());}
+    // Concurrent probes: exactly one claims it.
+    let latch=std::sync::Arc::new(StartLatch::new());
+    let claims:usize=(0..8).map(|_|{let l=latch.clone();std::thread::spawn(move||l.claim())})
+        .collect::<Vec<_>>().into_iter().map(|t|usize::from(t.join().unwrap())).sum();
+    assert_eq!(claims,1);
+}
+
+#[test]
+fn failed_pending_packets_quarantine_only_what_the_firmware_may_have_seen() {
+    // (in a batch fail_active drained, partly published and still pending, never in a batch)
+    struct Packet {published:bool,guard:bool}
+    let mut active=vec![Packet{published:true,guard:true}];
+    let mut pending=vec![Packet{published:true,guard:true},Packet{published:false,guard:true}];
+    // fail_active drains active before fail_pending: a check against active can't see these.
+    let drained:Vec<_>=active.drain(..).collect();
+    for p in pending.iter_mut() {if release_unpublished(p.published) {p.guard=false;}}
+    assert!(drained[0].guard);
+    assert!(pending[0].guard,"a partly published packet keeps its guard");
+    assert!(!pending[1].guard,"a never-published packet releases its guard");
+}
+
+#[test]
+fn the_start_pins_the_module_before_claiming_and_fails_closed() {
+    use std::cell::Cell;
+    let latch=StartLatch::new();
+    let refs=Cell::new(1i32); // the module's base reference
+    let pin_ok=||{refs.set(refs.get()+1);true};
+    let unpin=||refs.set(refs.get()-1);
+    // rmmod has begun: no pin, no claim, no start, and the reference count is untouched.
+    assert_eq!(claim_start(&latch,||false,unpin),Err(StartRefusal::Removing));
+    assert_eq!(refs.get(),1);
+    // The first live start wins and keeps its pin.
+    assert_eq!(claim_start(&latch,pin_ok,unpin),Ok(()));
+    assert_eq!(refs.get(),2);
+    // A later start (rebind, retry) loses the claim and drops only its own pin.
+    assert_eq!(claim_start(&latch,pin_ok,unpin),Err(StartRefusal::AlreadyStarted));
+    assert_eq!(refs.get(),2);
+    // A later start racing rmmod still claims nothing.
+    assert_eq!(claim_start(&latch,||false,||panic!("nothing to unpin")),Err(StartRefusal::Removing));
+    assert_eq!(refs.get(),2);
+}
diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py
new file mode 100644
index 0000000000000..c8428d65dbf0d
--- /dev/null
+++ b/tools/asahi/m3-fence-latency.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Join existing M3 timing logs. Input must be one boot, captured without gaps."""
+import collections
+import json
+import math
+import re
+import sys
+
+milestones = {}
+packets = collections.defaultdict(list)
+fences = []
+invalid = collections.Counter()
+for line in open(sys.argv[1]):
+    fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)}
+    if 'M3_BATCH_MILESTONE ' in line:
+        key = (fields['vm'], fields['publication_tick'])
+        if key in milestones:
+            invalid['duplicate_milestone'] += 1
+        milestones[key] = fields
+    elif 'M3_BATCH_PACKET ' in line:
+        packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command']))
+    elif 'M3_PACKET_FENCE ' in line:
+        fences.append(fields)
+values = []
+previous_signal = {}
+for f in sorted(fences, key=lambda f: f['signal_before_tick']):
+    identity = (f['vm'], f['packet'])
+    previous = previous_signal.get(identity, 0)
+    previous_signal[identity] = f['signal_after_tick']
+    if f['status'] != 0:
+        invalid['error_fence'] += 1
+        continue
+    linked = packets.get((f['vm'], f['packet']), [])
+    # Pointer reuse is disambiguated by the completed fence's physical counter.
+    candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1]
+    if not candidates:
+        invalid['missing_final_command'] += 1
+        continue
+    final_tick, _ = max(candidates)
+    batch = milestones.get((f['vm'], final_tick))
+    if not batch or batch['mixed'] or batch['render_end_tick'] <= 0:
+        invalid['missing_or_non_render_end'] += 1
+        continue
+    end = batch['render_end_tick']
+    before, after = f['signal_before_tick'], f['signal_after_tick']
+    if not (final_tick <= end <= before <= after):
+        invalid['invalid_counter_order'] += 1
+        continue
+    # GPU execution end is earlier than full coherent retirement; this includes
+    # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done.
+    values.append(((before - end) / 24, (after - end) / 24))
+
+def percentiles(data):
+    ordered = sorted(data)
+    return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {}
+
+print(json.dumps({'render_fences': len(values), 'invalid': invalid,
+                  'gpu_end_to_signal_before_us': percentiles([a for a, b in values]),
+                  'gpu_end_to_signal_after_us': percentiles([b for a, b in values]),
+                  'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2))
+if not values:
+    sys.exit('No valid render fences: do not report latency from this capture')
diff --git a/tools/asahi/m3-firmware-tests.rs b/tools/asahi/m3-firmware-tests.rs
new file mode 100644
index 0000000000000..90539441d623e
--- /dev/null
+++ b/tools/asahi/m3-firmware-tests.rs
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Host tests of production firmware identity and normalization.
+#![allow(dead_code)]
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"]
+pub(crate) mod m3_profile;
+#[path = "../../drivers/gpu/drm/asahi/m3/agx_resources.rs"]
+pub(crate) mod resources;
+mod m3 {
+    pub(crate) use crate::firmware as m3_firmware;
+    pub(crate) use crate::m3_profile;
+    pub(crate) use crate::resources as m3_resources;
+}
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_firmware.rs"]
+pub(crate) mod firmware;
diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs
index fd4dc90489e81..0d6d4e5841194 100644
--- a/tools/asahi/m3-runtime-tests.rs
+++ b/tools/asahi/m3-runtime-tests.rs
@@ -15,3 +15,97 @@ mod m3_init_storage;
 mod m3_thermal_policy;
 #[path = "../../drivers/gpu/drm/asahi/m3/t8122_hwdata.rs"]
 mod t8122_hwdata;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_init_26.rs"]
+mod m3_init_26;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"]
+mod m3_profile;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_mmio_26.rs"]
+mod m3_mmio_26;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_queue_layout.rs"]
+mod m3_queue_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_pool_layout.rs"]
+mod m3_pool_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_layout.rs"]
+mod m3_compute_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_channel_format.rs"]
+mod m3_channel_format;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs"]
+mod m3_compute_sequence;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_state_layout.rs"]
+mod m3_state_layout;
+
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"]
+mod m3_compute_storage;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs"]
+mod m3_parameter_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_sync_layout.rs"]
+mod m3_sync_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/agx_render.rs"]
+mod agx_render;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/agx_render_state.rs"]
+mod agx_render_state;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_scene_layout.rs"]
+mod m3_scene_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_command_layout.rs"]
+mod m3_command_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_render_sequence.rs"]
+mod m3_render_sequence;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_tiler_command.rs"]
+mod m3_tiler_command;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_pass_layout.rs"]
+mod m3_pass_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_shared_layout.rs"]
+mod m3_shared_layout;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_fragment_command.rs"]
+mod m3_fragment_command;
+
+#[path = "../../drivers/gpu/drm/asahi/m3/agx_attachments.rs"]
+mod agx_attachments;
+
+#[test]
+fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() {
+    use m3_pass_layout::{Access, Space};
+    use m3_profile::Profile;
+    use m3_shared_layout as shared;
+    for index in 0..shared::COUNT {
+        let original = shared::allocation(index).unwrap();
+        let legacy = shared::allocation_for(Profile::Legacy, index).unwrap();
+        assert_eq!(legacy, original);
+        let extended = shared::allocation_for(Profile::Extended, index).unwrap();
+        if [shared::PARAMETER_PAGE_LIST, shared::PARAMETER_BLOCK_LIST].contains(&index) {
+            assert_eq!(extended.space, Space::ClientGpu);
+            assert_eq!(extended.address, original.address);
+            assert_eq!(extended.size, original.size);
+            assert_eq!(extended.access, Access::GpuCachedRw);
+            assert_eq!(extended.access.pte(), 0xc0000000000080);
+            assert_eq!(original.access, Access::GpuFirmwareCachedRw);
+        }
+        if extended.space == Space::ClientGpu {
+            assert_ne!(extended.access, Access::GpuFirmwareCachedRw);
+        }
+    }
+}
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"]
+mod m3_completion_policy;
diff --git a/tools/asahi/m3-uapi-fixture.c b/tools/asahi/m3-uapi-fixture.c
new file mode 100644
index 0000000000000..9ee0994096009
--- /dev/null
+++ b/tools/asahi/m3-uapi-fixture.c
@@ -0,0 +1,363 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifndef ASAHI_HEADER
+#define ASAHI_HEADER "drm/asahi_drm.h"
+#endif
+#include ASAHI_HEADER
+
+static_assert(sizeof(struct drm_asahi_cmd_header) == 8, "command header");
+static_assert(sizeof(struct drm_asahi_attachment) == 24, "attachment");
+static_assert(sizeof(struct drm_asahi_timestamps) == 16, "timestamps");
+static_assert(offsetof(struct drm_asahi_cmd_compute, ts_end_copies) == 64,
+	      "compute prefix");
+static_assert(offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies) == 256,
+	      "render prefix");
+
+static int emit(unsigned int type, const void *payload, size_t size)
+{
+	struct drm_asahi_cmd_header header = {
+		.cmd_type = type,
+		.size = size,
+		.vdm_barrier = DRM_ASAHI_BARRIER_NONE,
+		.cdm_barrier = DRM_ASAHI_BARRIER_NONE,
+	};
+
+	return fwrite(&header, sizeof(header), 1, stdout) != 1 ||
+	       fwrite(payload, size, 1, stdout) != 1;
+}
+
+/* Named C offsets are exported independently of the Rust decoder. */
+static void layout(void)
+{
+	printf("profile.params_required %zu 0\n",
+	       offsetof(struct drm_asahi_params_global, gpu_hal_generation) + sizeof(__u32));
+	printf("profile.vm_create_size %zu 0\n", sizeof(struct drm_asahi_vm_create));
+	printf("profile.hal200 %u 0\n", (unsigned int)DRM_ASAHI_GPU_HAL_200);
+	printf("profile.features %u 0\n",
+	       DRM_ASAHI_FEATURE_VM_STATUS | DRM_ASAHI_FEATURE_SCHEDULED_QUEUES);
+	printf("header %zu 0\n", sizeof(struct drm_asahi_cmd_header));
+	printf("attachment %zu 0\n", sizeof(struct drm_asahi_attachment));
+	printf("timestamps %zu 0\n", sizeof(struct drm_asahi_timestamps));
+	printf("cmd_render.flags %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, flags),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->flags));
+	printf("cmd_render.isp_zls_pixels %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_zls_pixels),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_zls_pixels));
+	printf("cmd_render.vdm_ctrl_stream_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, vdm_ctrl_stream_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->vdm_ctrl_stream_base));
+	printf("cmd_render.vertex_helper.binary %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, vertex_helper.binary),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.binary));
+	printf("cmd_render.vertex_helper.cfg %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, vertex_helper.cfg),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.cfg));
+	printf("cmd_render.vertex_helper.data %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, vertex_helper.data),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.data));
+	printf("cmd_render.fragment_helper.binary %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, fragment_helper.binary),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.binary));
+	printf("cmd_render.fragment_helper.cfg %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, fragment_helper.cfg),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.cfg));
+	printf("cmd_render.fragment_helper.data %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, fragment_helper.data),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.data));
+	printf("cmd_render.isp_scissor_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_scissor_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_scissor_base));
+	printf("cmd_render.isp_dbias_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_dbias_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_dbias_base));
+	printf("cmd_render.isp_oclqry_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_oclqry_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_oclqry_base));
+	printf("cmd_render.depth.base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, depth.base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->depth.base));
+	printf("cmd_render.depth.comp_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, depth.comp_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_base));
+	printf("cmd_render.depth.stride %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, depth.stride),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->depth.stride));
+	printf("cmd_render.depth.comp_stride %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, depth.comp_stride),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_stride));
+	printf("cmd_render.stencil.base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, stencil.base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->stencil.base));
+	printf("cmd_render.stencil.comp_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, stencil.comp_base),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_base));
+	printf("cmd_render.stencil.stride %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, stencil.stride),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->stencil.stride));
+	printf("cmd_render.stencil.comp_stride %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, stencil.comp_stride),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_stride));
+	printf("cmd_render.zls_ctrl %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, zls_ctrl),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->zls_ctrl));
+	printf("cmd_render.ppp_multisamplectl %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ppp_multisamplectl),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ppp_multisamplectl));
+	printf("cmd_render.sampler_heap %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, sampler_heap),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->sampler_heap));
+	printf("cmd_render.ppp_ctrl %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ppp_ctrl),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ppp_ctrl));
+	printf("cmd_render.width_px %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, width_px),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->width_px));
+	printf("cmd_render.height_px %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, height_px),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->height_px));
+	printf("cmd_render.layers %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, layers),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->layers));
+	printf("cmd_render.sampler_count %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, sampler_count),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->sampler_count));
+	printf("cmd_render.utile_width_px %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, utile_width_px),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->utile_width_px));
+	printf("cmd_render.utile_height_px %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, utile_height_px),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->utile_height_px));
+	printf("cmd_render.samples %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, samples),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->samples));
+	printf("cmd_render.sample_size_B %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, sample_size_B),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->sample_size_B));
+	printf("cmd_render.isp_merge_upper_x %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_merge_upper_x),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_x));
+	printf("cmd_render.isp_merge_upper_y %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_merge_upper_y),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_y));
+	printf("cmd_render.bg.usc %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg.usc),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg.usc));
+	printf("cmd_render.bg.rsrc_spec %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg.rsrc_spec),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg.rsrc_spec));
+	printf("cmd_render.bg_rsrc_spec_hi %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg_rsrc_spec_hi),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg_rsrc_spec_hi));
+	printf("cmd_render.eot.usc %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, eot.usc),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->eot.usc));
+	printf("cmd_render.eot.rsrc_spec %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, eot.rsrc_spec),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->eot.rsrc_spec));
+	printf("cmd_render.bg_eot_rsrc_spec_hi %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg_eot_rsrc_spec_hi),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_rsrc_spec_hi));
+	printf("cmd_render.partial_bg.usc %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, partial_bg.usc),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.usc));
+	printf("cmd_render.partial_bg.rsrc_spec %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, partial_bg.rsrc_spec),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.rsrc_spec));
+	printf("cmd_render.bg_partial_rsrc_spec_hi %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg_partial_rsrc_spec_hi),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg_partial_rsrc_spec_hi));
+	printf("cmd_render.partial_eot.usc %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, partial_eot.usc),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.usc));
+	printf("cmd_render.partial_eot.rsrc_spec %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, partial_eot.rsrc_spec),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.rsrc_spec));
+	printf("cmd_render.bg_eot_partial_rsrc_spec_hi %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, bg_eot_partial_rsrc_spec_hi),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_partial_rsrc_spec_hi));
+	printf("cmd_render.isp_bgobjdepth %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_bgobjdepth),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjdepth));
+	printf("cmd_render.isp_bgobjvals %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, isp_bgobjvals),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjvals));
+	printf("cmd_render.ts_vtx.start.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_vtx.start.handle),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.handle));
+	printf("cmd_render.ts_vtx.start.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_vtx.start.offset),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.offset));
+	printf("cmd_render.ts_vtx.end.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_vtx.end.handle),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.handle));
+	printf("cmd_render.ts_vtx.end.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_vtx.end.offset),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.offset));
+	printf("cmd_render.ts_frag.start.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_frag.start.handle),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.handle));
+	printf("cmd_render.ts_frag.start.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_frag.start.offset),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.offset));
+	printf("cmd_render.ts_frag.end.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_frag.end.handle),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.handle));
+	printf("cmd_render.ts_frag.end.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_frag.end.offset),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.offset));
+	printf("cmd_compute.flags %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, flags),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->flags));
+	printf("cmd_compute.sampler_count %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, sampler_count),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_count));
+	printf("cmd_compute.cdm_ctrl_stream_base %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_base),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_base));
+	printf("cmd_compute.cdm_ctrl_stream_end %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_end),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_end));
+	printf("cmd_compute.sampler_heap %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, sampler_heap),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_heap));
+	printf("cmd_compute.helper.binary %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, helper.binary),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->helper.binary));
+	printf("cmd_compute.helper.cfg %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, helper.cfg),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->helper.cfg));
+	printf("cmd_compute.helper.data %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, helper.data),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->helper.data));
+	printf("cmd_compute.ts.start.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, ts.start.handle),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.handle));
+	printf("cmd_compute.ts.start.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, ts.start.offset),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.offset));
+	printf("cmd_compute.ts.end.handle %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, ts.end.handle),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.handle));
+	printf("cmd_compute.ts.end.offset %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, ts.end.offset),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.offset));
+	printf("attachment.pointer %zu %zu\n",
+	       offsetof(struct drm_asahi_attachment, pointer),
+	       sizeof(((struct drm_asahi_attachment *)0)->pointer));
+	printf("attachment.size %zu %zu\n",
+	       offsetof(struct drm_asahi_attachment, size),
+	       sizeof(((struct drm_asahi_attachment *)0)->size));
+	printf("cmd_render.ts_frag_end_copies %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies),
+	       sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag_end_copies));
+	printf("cmd_compute.ts_end_copies %zu %zu\n",
+	       offsetof(struct drm_asahi_cmd_compute, ts_end_copies),
+	       sizeof(((struct drm_asahi_cmd_compute *)0)->ts_end_copies));
+}
+
+int main(int argc, char **argv)
+{
+	struct drm_asahi_attachment attachment = {0};
+
+	attachment.pointer = 0xcdb09376593c1f02ULL;
+	attachment.size = 0xceb194775a3d2003ULL;
+
+	struct drm_asahi_cmd_render render = {0};
+
+	render.flags = 0x5b3e2104;
+	render.isp_zls_pixels = 0x5c3f2205;
+	render.vdm_ctrl_stream_base = 0xd1b4977a5d402306ULL;
+	render.vertex_helper.binary = 0x5e412407;
+	render.vertex_helper.cfg = 0x5f422508;
+	render.vertex_helper.data = 0xd4b79a7d60432609ULL;
+	render.fragment_helper.binary = 0x6144270a;
+	render.fragment_helper.cfg = 0x6245280b;
+	render.fragment_helper.data = 0xd7ba9d806346290cULL;
+	render.isp_scissor_base = 0xd8bb9e8164472a0dULL;
+	render.isp_dbias_base = 0xd9bc9f8265482b0eULL;
+	render.isp_oclqry_base = 0xdabda08366492c0fULL;
+	render.depth.base = 0xdbbea184674a2d10ULL;
+	render.depth.comp_base = 0xdcbfa285684b2e11ULL;
+	render.depth.stride = 0x694c2f12;
+	render.depth.comp_stride = 0x6a4d3013;
+	render.stencil.base = 0xdfc2a5886b4e3114ULL;
+	render.stencil.comp_base = 0xe0c3a6896c4f3215ULL;
+	render.stencil.stride = 0x6d503316;
+	render.stencil.comp_stride = 0x6e513417;
+	render.zls_ctrl = 0xe3c6a98c6f523518ULL;
+	render.ppp_multisamplectl = 0xe4c7aa8d70533619ULL;
+	render.sampler_heap = 0xe5c8ab8e7154371aULL;
+	render.ppp_ctrl = 0x7255381b;
+	render.width_px = 0x391c;
+	render.height_px = 0x3a1d;
+	render.layers = 0x3b1e;
+	render.sampler_count = 0x3c1f;
+	render.utile_width_px = 0x20;
+	render.utile_height_px = 0x21;
+	render.samples = 0x22;
+	render.sample_size_B = 0x23;
+	render.isp_merge_upper_x = 0x7b5e4124;
+	render.isp_merge_upper_y = 0x7c5f4225;
+	render.bg.usc = 0x7d604326;
+	render.bg.rsrc_spec = 0x7e614427;
+	render.bg_rsrc_spec_hi = 0x7f624528;
+	render.eot.usc = 0x80634629;
+	render.eot.rsrc_spec = 0x8164472a;
+	render.bg_eot_rsrc_spec_hi = 0x8265482b;
+	render.partial_bg.usc = 0x8366492c;
+	render.partial_bg.rsrc_spec = 0x84674a2d;
+	render.bg_partial_rsrc_spec_hi = 0x85684b2e;
+	render.partial_eot.usc = 0x86694c2f;
+	render.partial_eot.rsrc_spec = 0x876a4d30;
+	render.bg_eot_partial_rsrc_spec_hi = 0x886b4e31;
+	render.isp_bgobjdepth = 0x896c4f32;
+	render.isp_bgobjvals = 0x8a6d5033;
+	render.ts_vtx.start.handle = 0x8b6e5134;
+	render.ts_vtx.start.offset = 0x8c6f5235;
+	render.ts_vtx.end.handle = 0x8d705336;
+	render.ts_vtx.end.offset = 0x8e715437;
+	render.ts_frag.start.handle = 0x8f725538;
+	render.ts_frag.start.offset = 0x90735639;
+	render.ts_frag.end.handle = 0x9174573a;
+	render.ts_frag.end.offset = 0x9275583b;
+
+	struct drm_asahi_cmd_compute compute = {0};
+
+	compute.flags = 0x9376593c;
+	compute.sampler_count = 0x94775a3d;
+	compute.cdm_ctrl_stream_base = 0xbeccfb295785b3eULL;
+	compute.cdm_ctrl_stream_end = 0xcedd0b396795c3fULL;
+	compute.sampler_heap = 0xdeed1b4977a5d40ULL;
+	compute.helper.binary = 0x987b5e41;
+	compute.helper.cfg = 0x997c5f42;
+	compute.helper.data = 0x10f1d4b79a7d6043ULL;
+	compute.ts.start.handle = 0x9b7e6144;
+	compute.ts.start.offset = 0x9c7f6245;
+	compute.ts.end.handle = 0x9d806346;
+	compute.ts.end.offset = 0x9e816447;
+
+	size_t render_size =
+		offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies);
+	size_t compute_size =
+		offsetof(struct drm_asahi_cmd_compute, ts_end_copies);
+
+	if (argc == 2 && !strcmp(argv[1], "--layout")) {
+		layout();
+		return 0;
+	} else if (argc == 2 && !strcmp(argv[1], "--full")) {
+		render_size = sizeof(render);
+		compute_size = sizeof(compute);
+	} else if (argc != 1) {
+		return 1;
+	}
+
+	return emit(DRM_ASAHI_SET_VERTEX_ATTACHMENTS, &attachment,
+		    sizeof(attachment)) ||
+	       emit(DRM_ASAHI_CMD_RENDER, &render, render_size) ||
+	       emit(DRM_ASAHI_CMD_COMPUTE, &compute, compute_size);
+}
diff --git a/tools/asahi/m3-uapi-tests.rs b/tools/asahi/m3-uapi-tests.rs
new file mode 100644
index 0000000000000..4dc6012f5213e
--- /dev/null
+++ b/tools/asahi/m3-uapi-tests.rs
@@ -0,0 +1,832 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+//! Offline tests of the production parser. All addresses are synthetic.
+#![allow(dead_code)]
+
+#[path = "../../drivers/gpu/drm/asahi/m3/t8122_admission.rs"]
+mod admission;
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"]
+mod m3_profile;
+#[path = "../../drivers/gpu/drm/asahi/m3/agx_uapi.rs"]
+mod uapi;
+use uapi::*;
+
+fn frame(kind: u16, payload: &[u8]) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    for word in [
+        kind,
+        payload.len() as u16,
+        UAPI_BARRIER_NONE,
+        UAPI_BARRIER_NONE,
+    ] {
+        bytes.extend_from_slice(&word.to_le_bytes());
+    }
+    bytes.extend_from_slice(payload);
+    bytes
+}
+
+fn parse(bytes: &[u8]) -> Result<u32, UapiTranslateError> {
+    let mut parser = UapiCommandParser::new(bytes);
+    while parser.next_hardware()?.is_some() {}
+    parser.finish()
+}
+
+// Each offset below is checked against offsetof of the corresponding C member.
+fn layout() -> &'static std::collections::BTreeMap<String, (usize, usize)> {
+    static LAYOUT: std::sync::OnceLock<std::collections::BTreeMap<String, (usize, usize)>> =
+        std::sync::OnceLock::new();
+    LAYOUT.get_or_init(|| {
+        let path = std::env::var("ASAHI_UAPI_LAYOUT").expect("C layout path");
+        std::fs::read_to_string(path)
+            .unwrap()
+            .lines()
+            .map(|line| {
+                let words: Vec<_> = line.split_whitespace().collect();
+                (
+                    words[0].to_string(),
+                    (words[1].parse().unwrap(), words[2].parse().unwrap()),
+                )
+            })
+            .collect()
+    })
+}
+
+fn field(raw: &[u8], name: &str, offset: usize, width: usize, actual: u64) {
+    assert_eq!(layout()[name], (offset, width), "C offset/width for {name}");
+    let mut expected = [0u8; 8];
+    for (i, byte) in expected[..width].iter_mut().enumerate() {
+        *byte = raw.get(offset + i).copied().unwrap_or(0);
+    }
+    assert_eq!(actual, u64::from_le_bytes(expected), "decoded {name}");
+}
+
+fn check_render(p: UapiRenderCommand, raw: &[u8]) {
+    field(raw, "cmd_render.flags", 0, 4, p.flags as u64);
+    field(
+        raw,
+        "cmd_render.isp_zls_pixels",
+        4,
+        4,
+        p.depth_dimensions as u64,
+    );
+    field(
+        raw,
+        "cmd_render.vdm_ctrl_stream_base",
+        8,
+        8,
+        p.vdm_base as u64,
+    );
+    field(
+        raw,
+        "cmd_render.vertex_helper.binary",
+        16,
+        4,
+        p.vertex_helper.binary as u64,
+    );
+    field(
+        raw,
+        "cmd_render.vertex_helper.cfg",
+        20,
+        4,
+        p.vertex_helper.config as u64,
+    );
+    field(
+        raw,
+        "cmd_render.vertex_helper.data",
+        24,
+        8,
+        p.vertex_helper.data as u64,
+    );
+    field(
+        raw,
+        "cmd_render.fragment_helper.binary",
+        32,
+        4,
+        p.fragment_helper.binary as u64,
+    );
+    field(
+        raw,
+        "cmd_render.fragment_helper.cfg",
+        36,
+        4,
+        p.fragment_helper.config as u64,
+    );
+    field(
+        raw,
+        "cmd_render.fragment_helper.data",
+        40,
+        8,
+        p.fragment_helper.data as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_scissor_base",
+        48,
+        8,
+        p.scissor_base as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_dbias_base",
+        56,
+        8,
+        p.depth_bias_base as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_oclqry_base",
+        64,
+        8,
+        p.occlusion_query_base as u64,
+    );
+    field(raw, "cmd_render.depth.base", 72, 8, p.depth.base as u64);
+    field(
+        raw,
+        "cmd_render.depth.comp_base",
+        80,
+        8,
+        p.depth.compression_base as u64,
+    );
+    field(raw, "cmd_render.depth.stride", 88, 4, p.depth.stride as u64);
+    field(
+        raw,
+        "cmd_render.depth.comp_stride",
+        92,
+        4,
+        p.depth.compression_stride as u64,
+    );
+    field(raw, "cmd_render.stencil.base", 96, 8, p.stencil.base as u64);
+    field(
+        raw,
+        "cmd_render.stencil.comp_base",
+        104,
+        8,
+        p.stencil.compression_base as u64,
+    );
+    field(
+        raw,
+        "cmd_render.stencil.stride",
+        112,
+        4,
+        p.stencil.stride as u64,
+    );
+    field(
+        raw,
+        "cmd_render.stencil.comp_stride",
+        116,
+        4,
+        p.stencil.compression_stride as u64,
+    );
+    field(raw, "cmd_render.zls_ctrl", 120, 8, p.zls_control as u64);
+    field(
+        raw,
+        "cmd_render.ppp_multisamplectl",
+        128,
+        8,
+        p.multisample_control as u64,
+    );
+    field(
+        raw,
+        "cmd_render.sampler_heap",
+        136,
+        8,
+        p.sampler_heap as u64,
+    );
+    field(raw, "cmd_render.ppp_ctrl", 144, 4, p.ppp_control as u64);
+    field(raw, "cmd_render.width_px", 148, 2, p.width as u64);
+    field(raw, "cmd_render.height_px", 150, 2, p.height as u64);
+    field(raw, "cmd_render.layers", 152, 2, p.layers as u64);
+    field(
+        raw,
+        "cmd_render.sampler_count",
+        154,
+        2,
+        p.sampler_count as u64,
+    );
+    field(
+        raw,
+        "cmd_render.utile_width_px",
+        156,
+        1,
+        p.utile_width as u64,
+    );
+    field(
+        raw,
+        "cmd_render.utile_height_px",
+        157,
+        1,
+        p.utile_height as u64,
+    );
+    field(raw, "cmd_render.samples", 158, 1, p.samples as u64);
+    field(
+        raw,
+        "cmd_render.sample_size_B",
+        159,
+        1,
+        p.sample_size as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_merge_upper_x",
+        160,
+        4,
+        p.merge_upper_x as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_merge_upper_y",
+        164,
+        4,
+        p.merge_upper_y as u64,
+    );
+    field(raw, "cmd_render.bg.usc", 168, 4, p.background.usc as u64);
+    field(
+        raw,
+        "cmd_render.bg.rsrc_spec",
+        172,
+        4,
+        (p.background.resource_spec & 0xffffffff) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.bg_rsrc_spec_hi",
+        248,
+        4,
+        (p.background.resource_spec >> 32) as u64,
+    );
+    field(raw, "cmd_render.eot.usc", 176, 4, p.end_of_tile.usc as u64);
+    field(
+        raw,
+        "cmd_render.eot.rsrc_spec",
+        180,
+        4,
+        (p.end_of_tile.resource_spec & 0xffffffff) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.bg_eot_rsrc_spec_hi",
+        240,
+        4,
+        (p.end_of_tile.resource_spec >> 32) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.partial_bg.usc",
+        184,
+        4,
+        p.partial_background.usc as u64,
+    );
+    field(
+        raw,
+        "cmd_render.partial_bg.rsrc_spec",
+        188,
+        4,
+        (p.partial_background.resource_spec & 0xffffffff) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.bg_partial_rsrc_spec_hi",
+        252,
+        4,
+        (p.partial_background.resource_spec >> 32) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.partial_eot.usc",
+        192,
+        4,
+        p.partial_end_of_tile.usc as u64,
+    );
+    field(
+        raw,
+        "cmd_render.partial_eot.rsrc_spec",
+        196,
+        4,
+        (p.partial_end_of_tile.resource_spec & 0xffffffff) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.bg_eot_partial_rsrc_spec_hi",
+        244,
+        4,
+        (p.partial_end_of_tile.resource_spec >> 32) as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_bgobjdepth",
+        200,
+        4,
+        p.depth_clear as u64,
+    );
+    field(
+        raw,
+        "cmd_render.isp_bgobjvals",
+        204,
+        4,
+        p.stencil_clear as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_vtx.start.handle",
+        208,
+        4,
+        p.vertex_timestamps.start.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_vtx.start.offset",
+        212,
+        4,
+        p.vertex_timestamps.start.offset as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_vtx.end.handle",
+        216,
+        4,
+        p.vertex_timestamps.end.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_vtx.end.offset",
+        220,
+        4,
+        p.vertex_timestamps.end.offset as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_frag.start.handle",
+        224,
+        4,
+        p.fragment_timestamps.start.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_frag.start.offset",
+        228,
+        4,
+        p.fragment_timestamps.start.offset as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_frag.end.handle",
+        232,
+        4,
+        p.fragment_timestamps.end.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_render.ts_frag.end.offset",
+        236,
+        4,
+        p.fragment_timestamps.end.offset as u64,
+    );
+}
+
+fn check_compute(p: UapiComputeCommand, raw: &[u8]) {
+    field(raw, "cmd_compute.flags", 0, 4, p.flags as u64);
+    field(
+        raw,
+        "cmd_compute.sampler_count",
+        4,
+        4,
+        p.sampler_count as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.cdm_ctrl_stream_base",
+        8,
+        8,
+        p.control_stream_base as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.cdm_ctrl_stream_end",
+        16,
+        8,
+        p.control_stream_end as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.sampler_heap",
+        24,
+        8,
+        p.sampler_heap as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.helper.binary",
+        32,
+        4,
+        p.helper.binary as u64,
+    );
+    field(raw, "cmd_compute.helper.cfg", 36, 4, p.helper.config as u64);
+    field(raw, "cmd_compute.helper.data", 40, 8, p.helper.data as u64);
+    field(
+        raw,
+        "cmd_compute.ts.start.handle",
+        48,
+        4,
+        p.timestamps.start.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.ts.start.offset",
+        52,
+        4,
+        p.timestamps.start.offset as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.ts.end.handle",
+        56,
+        4,
+        p.timestamps.end.handle as u64,
+    );
+    field(
+        raw,
+        "cmd_compute.ts.end.offset",
+        60,
+        4,
+        p.timestamps.end.offset as u64,
+    );
+}
+
+fn fixture() -> Vec<u8> {
+    std::fs::read(std::env::var("ASAHI_UAPI_FIXTURE").expect("C fixture path")).unwrap()
+}
+
+#[test]
+fn c_header_producer_matches_production_decoder() {
+    let offsets = layout();
+    assert_eq!(offsets["header"].0, UAPI_COMMAND_HEADER_SIZE);
+    assert_eq!(offsets["attachment"].0, UAPI_ATTACHMENT_SIZE);
+    assert_eq!(offsets["timestamps"].0, 16);
+    assert_eq!(offsets["cmd_render.ts_frag_end_copies"].0, UAPI_RENDER_SIZE);
+    assert_eq!(offsets["cmd_compute.ts_end_copies"].0, UAPI_COMPUTE_SIZE);
+    let bytes = fixture();
+    let render_offset = 2 * UAPI_COMMAND_HEADER_SIZE + UAPI_ATTACHMENT_SIZE;
+    let render_size = u16::from_le_bytes(
+        bytes[render_offset - 6..render_offset - 4]
+            .try_into()
+            .unwrap(),
+    ) as usize;
+    let compute_offset = render_offset + render_size + UAPI_COMMAND_HEADER_SIZE;
+    let mut parser = UapiCommandParser::new(&bytes);
+    let Some(ParsedHardwareCommand::Render {
+        index,
+        header,
+        payload,
+        vertex_attachments,
+        fragment_attachments,
+    }) = parser.next_hardware().unwrap()
+    else {
+        panic!("render missing")
+    };
+    assert_eq!(index, 2);
+    assert_eq!(
+        header,
+        UapiCommandHeader {
+            command_type: 0,
+            size: render_size as u16,
+            render_barrier: UAPI_BARRIER_NONE,
+            compute_barrier: UAPI_BARRIER_NONE,
+        }
+    );
+    check_render(payload, &bytes[render_offset..render_offset + render_size]);
+    assert_eq!(vertex_attachments.count, 1);
+    let attachment = vertex_attachments.as_slice()[0];
+    field(
+        &bytes[8..32],
+        "attachment.pointer",
+        0,
+        8,
+        attachment.address,
+    );
+    field(&bytes[8..32], "attachment.size", 8, 8, attachment.size);
+    assert!(fragment_attachments.as_slice().is_empty());
+    let Some(ParsedHardwareCommand::Compute {
+        index,
+        header,
+        payload,
+        attachments,
+    }) = parser.next_hardware().unwrap()
+    else {
+        panic!("compute missing")
+    };
+    assert_eq!(index, 3);
+    assert_eq!(
+        header,
+        UapiCommandHeader {
+            command_type: 1,
+            size: (bytes.len() - compute_offset) as u16,
+            render_barrier: UAPI_BARRIER_NONE,
+            compute_barrier: UAPI_BARRIER_NONE,
+        }
+    );
+    check_compute(payload, &bytes[compute_offset..]);
+    assert!(attachments.as_slice().is_empty());
+    assert!(parser.next_hardware().unwrap().is_none());
+    assert_eq!(parser.finish(), Ok(2));
+}
+
+#[test]
+fn short_payloads_zero_extend_but_missing_bytes_are_rejected() {
+    // Every length includes cuts within each multi-byte field, for both kinds.
+    for (kind, size) in [(0, 256), (1, 64)] {
+        let supplied: Vec<u8> = (0..size).map(|i| (i % 254 + 1) as u8).collect();
+        for length in 0..=size {
+            let raw = &supplied[..length];
+            let bytes = frame(kind, raw);
+            let mut parser = UapiCommandParser::new(&bytes);
+            match parser.next_hardware().unwrap().unwrap() {
+                ParsedHardwareCommand::Render { payload, .. } => check_render(payload, raw),
+                ParsedHardwareCommand::Compute { payload, .. } => check_compute(payload, raw),
+            }
+            assert!(parser.next_hardware().unwrap().is_none());
+            assert_eq!(parser.finish(), Ok(1));
+            for cut in 1..bytes.len() {
+                assert!(parse(&bytes[..cut]).is_err());
+            }
+        }
+    }
+    assert_eq!(parse(&[]), Err(UapiTranslateError::HardwareCommandCount));
+}
+
+#[test]
+fn named_optional_timestamp_copies_are_refused() {
+    for (kind, name) in [
+        (0, "cmd_render.ts_frag_end_copies"),
+        (1, "cmd_compute.ts_end_copies"),
+    ] {
+        let (offset, size) = layout()[name];
+        let mut raw = vec![0; offset + size];
+        assert_eq!(parse(&frame(kind, &raw)), Ok(1));
+        for i in offset..raw.len() {
+            raw[i] = 1;
+            assert_eq!(
+                parse(&frame(kind, &raw)),
+                Err(UapiTranslateError::PayloadTail),
+                "{name} byte {i}"
+            );
+            raw[i] = 0;
+        }
+    }
+}
+
+#[test]
+fn unsupported_nonzero_extensions_never_disappear() {
+    for (kind, size) in [(0, UAPI_RENDER_SIZE), (1, UAPI_COMPUTE_SIZE)] {
+        for extra in [1, 8, 64, 128] {
+            let mut payload = vec![0; size + extra];
+            assert_eq!(parse(&frame(kind, &payload)), Ok(1));
+            for offset in size..payload.len() {
+                payload[offset] = 1;
+                assert_eq!(
+                    parse(&frame(kind, &payload)),
+                    Err(UapiTranslateError::PayloadTail)
+                );
+                payload[offset] = 0;
+            }
+        }
+    }
+    assert_eq!(
+        parse(&frame(0xffff, &[])),
+        Err(UapiTranslateError::CommandType)
+    );
+}
+
+#[test]
+fn attachment_shapes_counts_and_reserved_fields_are_checked() {
+    for kind in 2..=4 {
+        for size in 1..UAPI_ATTACHMENT_SIZE {
+            assert_eq!(
+                parse(&frame(kind, &vec![0; size])),
+                Err(UapiTranslateError::AttachmentPayloadSize)
+            );
+        }
+        let mut bytes = frame(kind, &vec![0; UAPI_ATTACHMENT_SIZE * UAPI_MAX_ATTACHMENTS]);
+        bytes.extend(frame(1, &[]));
+        assert_eq!(parse(&bytes), Ok(1));
+        assert_eq!(
+            parse(&frame(
+                kind,
+                &vec![0; UAPI_ATTACHMENT_SIZE * (UAPI_MAX_ATTACHMENTS + 1)]
+            )),
+            Err(UapiTranslateError::AttachmentCount)
+        );
+        for offset in 16..24 {
+            let mut payload = [0; UAPI_ATTACHMENT_SIZE];
+            payload[offset] = 1;
+            assert_eq!(
+                parse(&frame(kind, &payload)),
+                Err(UapiTranslateError::AttachmentFields)
+            );
+        }
+        let mut bytes = frame(kind, &[]);
+        bytes[4..6].copy_from_slice(&0u16.to_le_bytes());
+        assert_eq!(parse(&bytes), Err(UapiTranslateError::AttachmentBarrier));
+    }
+}
+
+#[test]
+fn attachment_lists_reach_their_commands() {
+    let entry = |pointer: u64, size: u64| {
+        let mut raw = [0u8; UAPI_ATTACHMENT_SIZE];
+        raw[..8].copy_from_slice(&pointer.to_le_bytes());
+        raw[8..16].copy_from_slice(&size.to_le_bytes());
+        raw
+    };
+    let vertex = entry(0x1_2345_6000, 0x1_0000_0040).to_vec();
+    let fragment = [
+        entry(0x2_0000_8000, 0x80),
+        entry(0x3_0000_c000, 0x2_0000_0100),
+    ]
+    .concat();
+    let compute = entry(0x4_0000_1000, 0x5_0000_0010).to_vec();
+    let check = |list: UapiAttachmentList, raw: &[u8]| {
+        assert_eq!(list.as_slice().len(), raw.len() / UAPI_ATTACHMENT_SIZE);
+        for (attachment, raw) in list.as_slice().iter().zip(raw.chunks(UAPI_ATTACHMENT_SIZE)) {
+            field(raw, "attachment.pointer", 0, 8, attachment.address);
+            field(raw, "attachment.size", 8, 8, attachment.size);
+        }
+    };
+
+    // Lists persist until replaced; an empty list clears only its own kind.
+    let mut bytes = frame(2, &vertex);
+    bytes.extend(frame(3, &fragment));
+    bytes.extend(frame(4, &compute));
+    bytes.extend(frame(0, &[]));
+    bytes.extend(frame(1, &[]));
+    bytes.extend(frame(3, &[]));
+    bytes.extend(frame(0, &[]));
+    let mut parser = UapiCommandParser::new(&bytes);
+    for expected_fragment in [&fragment[..], &[]] {
+        let Some(ParsedHardwareCommand::Render {
+            vertex_attachments,
+            fragment_attachments,
+            ..
+        }) = parser.next_hardware().unwrap()
+        else {
+            panic!("render missing")
+        };
+        check(vertex_attachments, &vertex);
+        check(fragment_attachments, expected_fragment);
+        if !expected_fragment.is_empty() {
+            let Some(ParsedHardwareCommand::Compute { attachments, .. }) =
+                parser.next_hardware().unwrap()
+            else {
+                panic!("compute missing")
+            };
+            check(attachments, &compute);
+        }
+    }
+    assert!(parser.next_hardware().unwrap().is_none());
+    assert_eq!(parser.finish(), Ok(3));
+}
+
+#[test]
+fn dependencies_count_hardware_commands_and_bound_the_submit() {
+    for field in [4, 6] {
+        let mut bytes = frame(1, &[]);
+        bytes[field..field + 2].copy_from_slice(&1u16.to_le_bytes());
+        assert_eq!(parse(&bytes), Err(UapiTranslateError::BarrierFuture));
+    }
+    let mut bytes = frame(2, &[]);
+    bytes.extend(frame(0, &[]));
+    let mut compute = frame(1, &[]);
+    compute[4..6].copy_from_slice(&1u16.to_le_bytes());
+    bytes.extend(compute);
+    assert_eq!(parse(&bytes), Ok(2));
+    let command = frame(1, &[]);
+    assert_eq!(
+        parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize)),
+        Ok(UAPI_MAX_HARDWARE_COMMANDS)
+    );
+    assert_eq!(
+        parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize + 1)),
+        Err(UapiTranslateError::HardwareCommandCount)
+    );
+}
+
+struct Region {
+    start: u64,
+    end: u64,
+    writable: bool,
+}
+impl GpuAddressSpace for Region {
+    fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool {
+        let Some(end) = address.checked_add(size) else {
+            return false;
+        };
+        address >= self.start && end <= self.end && (access == GpuAccess::Read || self.writable)
+    }
+}
+
+#[test]
+fn attachment_validation_rejects_readonly_zero_and_wrapped_ranges() {
+    let mut list = UapiAttachmentList::EMPTY;
+    list.count = 1;
+    let mut region = Region {
+        start: 0x4000,
+        end: 0x8000,
+        writable: false,
+    };
+    list.entries[0] = UapiAttachment {
+        address: 0x4000,
+        size: 0x4000,
+    };
+    assert_eq!(
+        validate_attachments(&region, &list),
+        Err(UapiTranslateError::AddressRange)
+    );
+    region.writable = true;
+    assert_eq!(validate_attachments(&region, &list), Ok(()));
+    for (address, size) in [(0x4000, 0), (0x4000, 0x4001), (u64::MAX, 2)] {
+        list.entries[0] = UapiAttachment { address, size };
+        assert_eq!(
+            validate_attachments(&region, &list),
+            Err(UapiTranslateError::AddressRange)
+        );
+    }
+}
+
+#[test]
+fn usc_windows_reject_misalignment_overflow_and_unmapped_programs() {
+    let queue = QueueUscWindow {
+        base: USC_WINDOW_SIZE,
+        user_start: 0x4000,
+        user_end: 2 * USC_WINDOW_SIZE,
+    };
+    assert_eq!(queue.validate(), Ok(()));
+    assert_eq!(
+        QueueUscWindow {
+            base: queue.base + 1,
+            ..queue
+        }
+        .validate(),
+        Err(UapiTranslateError::QueueUscAlignment)
+    );
+    assert_eq!(
+        QueueUscWindow {
+            base: u64::MAX & !(USC_WINDOW_SIZE - 1),
+            ..queue
+        }
+        .validate(),
+        Err(UapiTranslateError::QueueUscRange)
+    );
+    assert_eq!(
+        QueueUscWindow {
+            user_end: queue.user_end - 1,
+            ..queue
+        }
+        .validate(),
+        Err(UapiTranslateError::QueueUscRange)
+    );
+    let region = Region {
+        start: queue.base + 8,
+        end: queue.base + 12,
+        writable: false,
+    };
+    assert_eq!(queue.full_program(&region, 9), Ok(queue.base + 8));
+    assert_eq!(
+        queue.full_program(&region, 0),
+        Err(UapiTranslateError::ProgramOffset)
+    );
+    assert_eq!(
+        queue.full_program(&region, 16),
+        Err(UapiTranslateError::AddressRange)
+    );
+}
+
+#[test]
+fn unsupported_firmware_cannot_reach_the_existing_profile() {
+    let identity = (1, 1, 0x07022000, [0xff, 0, 0]);
+    assert!(admission::validate_identity(
+        &[14, 8, 3],
+        identity.0,
+        identity.1,
+        identity.2,
+        identity.3
+    )
+    .is_some());
+    for version in [&[26, 6, 2][..], &[14, 8][..], &[14, 8, 3, 0][..], &[][..]] {
+        assert!(admission::validate_identity(
+            version, identity.0, identity.1, identity.2, identity.3
+        )
+        .is_none());
+    }
+}
+
+#[test]
+fn firmware_profile_agrees_with_the_actual_c_header() {
+    assert_eq!(
+        layout()["profile.params_required"].0 as u64,
+        m3_profile::PARAMS_REQUIRED
+    );
+    assert_eq!(layout()["profile.vm_create_size"].0, 24);
+    assert_eq!(
+        layout()["profile.hal200"].0 as u32,
+        m3_profile::Profile::Extended.hal()
+    );
+    assert_eq!(layout()["profile.features"].0, 0x0a);
+}
diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py
new file mode 100644
index 0000000000000..1c5b4695c85f7
--- /dev/null
+++ b/tools/asahi/queue-priority-tests.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Compile the production queue admission gate with controlled credentials/backends."""
+from pathlib import Path
+import subprocess
+import tempfile
+root = Path(__file__).resolve().parents[2]
+s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text()
+a = s.index('        if data.flags != 0', s.index('pub(crate) fn queue_create('))
+b = s.index('        let queues_xa', a)
+gate = s[a:b]
+predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n    }', 1)[0]
+submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text()
+ordering = submit.split('        let urgency=', 1)[1].split('        Ok(Self', 1)[0]
+ordering = 'let urgency=' + ordering
+assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s
+unit = r'''
+#![allow(dead_code, non_upper_case_globals)]
+use std::any::Any;
+const EINVAL:i32=22; const EACCES:i32=13;
+macro_rules! cls_pr_debug { ($($arg:tt)*) => {} }
+mod uapi {
+ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3;
+ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2;
+}
+mod bindings { pub const CAP_SYS_NICE:u32=23;
+ pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) }
+ thread_local! {pub static CAP:std::cell::Cell<bool>=const {std::cell::Cell::new(false)};}
+}
+mod g17 {pub struct Gpu;}
+mod m3 {pub mod m3_drm {
+ #[derive(PartialEq)] pub enum Profile {Legacy, Extended}
+ pub struct Soc {pub chip_id:u32}
+ pub struct Backend {pub soc:Soc,pub profile:Profile}
+ impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }}
+}}
+struct Gpu(Box<dyn Any>); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}}
+struct Reg {gpu:Gpu}
+struct Data {flags:u32,priority:u32}
+struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}}
+fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) }
+mod sched {
+ #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel}
+}
+fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> {
+ ORDERING
+ Ok((urgency,priority))
+}
+fn main() {
+ use m3::m3_drm::*;
+ let mut n=0;
+ for backend in 0..5 { for cap in [false,true] { for master in [false,true] {
+  let gpu:Box<dyn Any>=match backend {
+   0=>Box::new(g17::Gpu),
+   1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}),
+   2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}),
+   3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}),
+   _=>Box::new(())};
+  let reg=Reg{gpu:Gpu(gpu)};
+  bindings::CAP.with(|c|c.set(cap));
+  for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] {
+   let expected=if flags!=0 || priority>=3 {Err(EINVAL)}
+    else if priority!=2 {Ok(())}
+    else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)}
+    else if cap||master {Ok(())} else {Err(EACCES)};
+   assert_eq!(admit(&reg,&Data{flags,priority},&File(master)),expected,
+    "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1;
+  }}
+ }}}
+ for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] {
+  assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected)));
+ }
+ assert_eq!(ordering(4),Err(EINVAL));
+ println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed");
+}
+'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering)
+with tempfile.TemporaryDirectory(prefix='queue-priority-') as d:
+    p = Path(d) / 'gate.rs'
+    p.write_text(unit)
+    for enabled in [False, True]:
+        subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True)
+        # Without M3 compiled only the G17 backend is admitted.
+        subprocess.run([str(p.with_suffix(''))], check=True)
+print('M3 enabled/disabled gate compilation passed')
diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py
index 5803cdfa8aada..6900b7fe82f5f 100755
--- a/tools/asahi/run-host-tests.py
+++ b/tools/asahi/run-host-tests.py
@@ -53,9 +53,13 @@ def mutations(flag, names, base, expect=KILL):
 
 # test file -> [(arguments, expectation)]. '{out}' is a fresh directory.
 ASAHI_TESTS = {
+    'queue-priority-tests.py': [([], PASS)],
     # The old revision reports a DART fault that clears before the second latch read as success.
     'ane-tm-transient-fault-tests.py': [([], PASS),
                                         (['--old-ref', 'b50066848322c6799c1433ee3a358aab2e9faabb'], PASS)],
+    'test-m3-power.py': [([], PASS)],
+    'test-m3-uapi.py': [([], PASS)],
+    'test-m3-mailbox.py': [([], PASS)],
     'test-j613-dcp-power.py': [([], PASS)],
     'aop-als-channel-tests.py': [([], PASS)],
     'aop-bootargs-tests.py': [([], PASS)],
@@ -195,6 +199,7 @@ def mutations(flag, names, base, expect=KILL):
 
 # Production G17 decoder vectors also build directly with rustc --test.
 ASAHI_RUST += ['g17-telemetry-tests.rs', 'legacy-frequency-tests.rs']
+ASAHI_RUST.append('m3-firmware-tests.rs')
 
 # Python that runs on the host but imports its neighbours, so it keeps the
 # script directory on sys.path (-E -s instead of -I).
@@ -211,6 +216,9 @@ def mutations(flag, names, base, expect=KILL):
 def jobs(work, conf, dtc):
     found = {p.name for p in ASAHI.iterdir()
              if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name}
+    # This Rust fixture is compiled and executed by the paired-header wrapper.
+    if 'test-m3-uapi.py' in ASAHI_TESTS:
+        found.discard('m3-uapi-tests.rs')
     known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL)
     missing = sorted(found - known)
     gone = sorted(name for name in known if not (ASAHI / name).is_file())
diff --git a/tools/asahi/test-m3-mailbox.py b/tools/asahi/test-m3-mailbox.py
new file mode 100644
index 0000000000000..167b0566b2e5d
--- /dev/null
+++ b/tools/asahi/test-m3-mailbox.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Exercise the production T8122 mailbox predicate with captured DTB properties.
+
+Usage: python3 tools/asahi/test-m3-mailbox.py [runtime.dtb daily.dtb]
+Requires dtc's fdtget. No target access or MMIO.
+"""
+import hashlib
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[2]
+board = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_board.rs').read_text()
+soc = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_soc.rs').read_text()
+t8122 = soc[soc.index('pub(crate) static T8122:'):]
+compatibles = re.search(r'mailbox_compatibles: &(\[.*?\]),', t8122, re.S)[1]
+irqs = re.search(r'mailbox_interrupts: &(\[.*?\]),', t8122, re.S)[1]
+base = re.search(r'mailbox: (0x[0-9a-fA-F_]+),', t8122)[1]
+regfn = re.search(r'fn expected_mailbox_reg\(.*?\n}', board, re.S)[0]
+predicate = board[board.index('    if !soc.mailbox_compatibles'):]
+predicate = predicate[:predicate.index('\n    {')].strip()[3:]
+
+
+def prop(dtb, node, name, kind):
+    result = subprocess.check_output(['fdtget', '-t', kind, str(dtb), node, name], text=True)
+    return [int(v, 16) for v in result.split()]
+
+
+def vec(values):
+    return 'vec![' + ','.join(map(str, values)) + ']'
+
+
+fixtures = []
+for argument in sys.argv[1:]:
+    dtb = Path(argument)
+    gpu = '/soc/gpu@292400000'
+    # Both retained captures use this mailbox path; verify the phandle link,
+    # not its capture-specific numeric value.
+    mbox = '/soc/mbox@292408000'
+    if prop(dtb, gpu, 'mboxes', 'x') != prop(dtb, mbox, 'phandle', 'x'):
+        raise RuntimeError('GPU mailbox phandle mismatch')
+    values = [prop(dtb, mbox, name, kind) for name, kind in
+              [('compatible', 'bx'), ('reg', 'x'), ('interrupt-names', 'bx'),
+               ('interrupts', 'x'), ('#mbox-cells', 'x')]]
+    if len(values[4]) != 1:
+        raise RuntimeError('bad #mbox-cells length')
+    fixtures.append('check(' + ','.join(vec(v) for v in values[:4]) + ',' + str(values[4][0]) + ');')
+    print(dtb.name, hashlib.sha256(dtb.read_bytes()).hexdigest())
+if not sys.argv[1:]:
+    # Public t8122-j613-dcp-25g83.dtsi and t8122-gpu.dtsi resource contract.
+    names = list(b'send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0')
+    for compat in (b'apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0',
+                   b'apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0'):
+        values = [list(compat), [2, 0x92408000, 0, 0x4000], names,
+                  [0,723,4,0,724,4,0,725,4,0,726,4]]
+        fixtures.append('check(' + ','.join(vec(v) for v in values) + ',0);')
+if len(fixtures) != 2:
+    raise SystemExit('supply runtime.dtb and daily.dtb')
+
+source = '''
+struct Windows { mailbox: u64 }
+struct Soc { mailbox_compatibles: &'static [&'static [u8]], mailbox_interrupts: &'static [u32], windows: Windows }
+''' + regfn + '''
+fn accepts(soc: &Soc, compat: &Vec<u8>, mbox_reg: &Vec<u32>, irq_names: &Vec<u8>, irqs: &Vec<u32>, cells: u32) -> bool {
+    !(''' + predicate + ''')
+}
+fn check(compat: Vec<u8>, reg: Vec<u32>, names: Vec<u8>, irqs: Vec<u32>, cells: u32) {
+    let soc = Soc { mailbox_compatibles: &''' + compatibles + ''', mailbox_interrupts: &''' + irqs + ''', windows: Windows { mailbox: ''' + base + ''' } };
+    let ok = |c: &Vec<u8>, r: &Vec<u32>, n: &Vec<u8>, i: &Vec<u32>, x| accepts(&soc,c,r,n,i,x);
+    assert!(ok(&compat,&reg,&names,&irqs,cells));
+    for i in 0..reg.len() { let mut v=reg.clone(); v[i]^=1; assert!(!ok(&compat,&v,&names,&irqs,cells)); }
+    for i in 0..irqs.len() { let mut v=irqs.clone(); v[i]^=1; assert!(!ok(&compat,&reg,&names,&v,cells)); }
+    assert!(!ok(&compat,&reg,&names,&irqs,1));
+    let mut v=names.clone(); v[0]^=1; assert!(!ok(&compat,&reg,&v,&irqs,cells));
+    for v in [b"apple,asc-mailbox-v4\\0".to_vec(), b"apple,t8122-agx-asc-mailbox\\0unknown\\0".to_vec(), b"apple,t6031-agx-asc-mailbox\\0".to_vec()] { assert!(!ok(&v,&reg,&names,&irqs,cells)); }
+    // Compatible lists are ordered: reversing either supported pair must fail.
+    for pair in soc.mailbox_compatibles {
+        let parts: Vec<_> = pair.split(|byte| *byte == 0).filter(|part| !part.is_empty()).collect();
+        if parts.len() == 2 {
+            let mut reversed = parts[1].to_vec();
+            reversed.push(0);
+            reversed.extend_from_slice(parts[0]);
+            reversed.push(0);
+            assert!(!ok(&reversed,&reg,&names,&irqs,cells));
+        }
+    }
+    for v in soc.mailbox_compatibles { assert!(ok(&v.to_vec(),&reg,&names,&irqs,cells)); }
+}
+fn main() {
+''' + '\n'.join(fixtures) + '\nprintln!("mailbox: both captures, all exact supported lists and resource/identity mutation controls PASS");\n}\n'
+with tempfile.TemporaryDirectory() as tmp:
+    path = Path(tmp)
+    (path / 'test.rs').write_text(source)
+    subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path / 'test.rs'), '-o', str(path / 'test')], check=True)
+    subprocess.run([str(path / 'test')], check=True)
+print('m3_board.rs', hashlib.sha256(board.encode()).hexdigest())
+print('m3_soc.rs', hashlib.sha256(soc.encode()).hexdigest())
diff --git a/tools/asahi/test-m3-power.py b/tools/asahi/test-m3-power.py
new file mode 100644
index 0000000000000..51bea088cd9db
--- /dev/null
+++ b/tools/asahi/test-m3-power.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Run the production controller writer and soft-float code without a GPU.
+
+Kernel allocation/DT adapters are replaced by host adapters. Public production
+power types and PStateTables are extracted as complete balanced Rust items,
+never restated by hand. Their input hashes are printed with the test evidence.
+"""
+from pathlib import Path
+import hashlib
+import subprocess
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[2]
+M3 = ROOT / "drivers/gpu/drm/asahi/m3"
+
+
+def item(source, anchor):
+    start = source.index(anchor)
+    opening = source.index("{", start)
+    depth = 1
+    end = opening + 1
+    while depth:
+        if source[end] == "{":
+            depth += 1
+        elif source[end] == "}":
+            depth -= 1
+        end += 1
+    return source[start:end]
+
+
+shim = r'''
+#![allow(dead_code)]
+extern crate self as kernel;
+pub type Error = i32;
+pub type Result<T = ()> = core::result::Result<T, Error>;
+pub const EINVAL: Error = 22;
+pub const EOVERFLOW: Error = 75;
+pub const ERANGE: Error = 34;
+pub const GFP_KERNEL: () = ();
+pub unsafe trait Zeroable {}
+#[derive(Debug)]
+pub struct KVec<T>(Vec<T>);
+impl<T> Default for KVec<T> { fn default() -> Self { Self(Vec::new()) } }
+impl<T> KVec<T> {
+    pub fn new() -> Self { Self::default() }
+    pub fn resize(&mut self, n: usize, value: T, _: ()) -> Result where T: Clone {
+        self.0.resize(n, value); Ok(())
+    }
+}
+impl<T> core::ops::Deref for KVec<T> { type Target = Vec<T>; fn deref(&self) -> &Vec<T> { &self.0 } }
+impl<T> core::ops::DerefMut for KVec<T> { fn deref_mut(&mut self) -> &mut Vec<T> { &mut self.0 } }
+pub mod prelude { pub use crate::{Error,Result,EINVAL,EOVERFLOW,ERANGE,GFP_KERNEL,KVec,Zeroable}; }
+pub mod of {
+    pub struct Property<'a>(pub &'a [u8]);
+    pub trait PropertyUnit: Sized { const UNIT_SIZE: usize; fn from_bytes(data: &[u8]) -> crate::Result<Self>; }
+    impl PropertyUnit for u32 {
+        const UNIT_SIZE: usize = 4;
+        fn from_bytes(data: &[u8]) -> crate::Result<Self> {
+            Ok(u32::from_be_bytes(data.try_into().map_err(|_| crate::EINVAL)?))
+        }
+    }
+    impl TryFrom<Property<'_>> for u32 {
+        type Error = crate::Error;
+        fn try_from(value: Property<'_>) -> crate::Result<Self> { Self::from_bytes(value.0) }
+    }
+}
+'''
+fixture = r'''
+#[cfg(test)] mod controller_tests {
+    use super::*;
+    use float::F32;
+    fn input() -> hw::PwrConfig {
+        let mut p = hw::PwrConfig::default();
+        p.perf_states = KVec((0..9).map(|i| hw::PState {
+            freq_hz: if i == 8 { 1_338_000_000 } else { i * 100_000_000 },
+            volt_mv: KVec(vec![if i == 0 { 125 } else { 450 + i * 50 }]), pwr_mw: i * 1000,
+        }).collect());
+        p.perf_base_pstate = 1;
+        p.power_sample_period = 8;
+        p.ppm_filter_time_constant_ms = 16;
+        p.avg_power_filter_tc_ms = 16;
+        p.pwr_filter_time_constant = 2;
+        p.perf_filter_time_constant = 2;
+        p.perf_filter_time_constant2 = 2;
+        p.avg_power_target_filter_tc = 2;
+        p.se_filter_time_constant = 2;
+        p.se_filter_time_constant_1 = 2;
+        p.pwr_min_duty_cycle = 30;
+        p.avg_power_min_duty_cycle = 20;
+        p
+    }
+    fn build(p: &hw::PwrConfig, sram: &[u32]) -> Result<m3_power_26::PowerImages> {
+        m3_power_26::PowerImages::new(p, sram, 0xffff_fc21_0340_0000,
+            0xffff_fc21_0350_0000, 0xffff_fc2f_fc00_0000, &[0; 53 * 40])
+    }
+    fn word(b: &[u8], at: usize) -> u32 { u32::from_le_bytes(b[at..at + 4].try_into().unwrap()) }
+    #[test] fn native_controller_bounds_are_distinct_from_outputs_and_hardware_state() {
+        let image = build(&input(), &[950_001; 9]).unwrap();
+        assert_eq!(image.hardware.len(), 0x2658);
+        assert_eq!(image.performance.len(), 0x6578);
+        assert_eq!(image.power.len(), 0x670);
+        assert_eq!(image.globals.len(), 0xe48);
+        assert_eq!(image.maximum_khz, 1_338_000);
+        assert_eq!(word(&image.performance, 0x2c), 1);
+        assert_eq!(word(&image.performance, 0x30), 1);
+        assert_eq!(word(&image.performance, 0x38), 0);
+        assert_eq!(word(&image.performance, 0x9f0), 30);
+        assert_eq!(word(&image.performance, 0xa38), 800);
+        assert_eq!(word(&image.performance, 0xb78), 100);
+        assert_eq!(word(&image.performance, 0xb7c), 800);
+        assert_eq!(word(&image.hardware, 0x1408 + 64), 951);
+        assert_eq!(word(&image.performance, 0x2268), 0x4248);
+        assert_eq!(word(&image.performance, 0x3b58), 0x4b48);
+    }
+    #[test] fn malformed_tunables_and_nonfinite_gains_are_rejected() {
+        for period in [0, 1001, u32::MAX] {
+            let mut p = input(); p.power_sample_period = period;
+            assert!(build(&p, &[950_000; 9]).is_err());
+        }
+        for bits in [0x7f800000, 0xff800000, 0x7fc00001] {
+            let mut p = input(); p.pwr_integral_gain = F32::from_bits(bits);
+            assert!(build(&p, &[950_000; 9]).is_err());
+        }
+        assert!(build(&input(), &[950_000; 8]).is_err());
+        assert!(build(&input(), &[100_000; 9]).is_err());
+        let mut p = input(); p.avg_power_target_filter_tc = u32::MAX;
+        assert!(build(&p, &[950_000; 9]).is_err());
+    }
+    #[test] fn dt_order_is_not_the_firmware_voltage_order() {
+        let p = input();
+        let expected = build(&p, &[950_000; 9]).unwrap();
+        let mut shuffled = input();
+        shuffled.perf_states.swap(2, 6);
+        let actual = build(&shuffled, &[950_000; 9]).unwrap();
+        assert_eq!(actual.hardware.0, expected.hardware.0);
+        assert_eq!(actual.performance.0, expected.performance.0);
+    }
+}
+'''
+
+
+def main():
+    hw = (M3 / "hw.rs").read_text()
+    init = (M3 / "initdata.rs").read_text()
+    code = shim
+    # Keep the production soft-float prefix verbatim. Its unrelated existing
+    # print-only TODO test lacks an import and is not part of this harness.
+    soft_float = (ROOT / "drivers/gpu/drm/asahi/float.rs").read_text()
+    code += "mod float {\n" + soft_float.split("// TODO: Make this an actual test", 1)[0] + "\n}\n"
+    code += "mod hw { use crate::{KVec,float::F32};\n"
+    for name in ["PState", "PowerZone", "CsAfrPwrConfig", "PwrConfig"]:
+        code += "#[derive(Default)]\n" + item(hw, "pub(crate) struct " + name + " {") + "\n"
+    code += item(hw, "impl PState {") + "\n"
+    code += "}\nmod initdata { use crate::{prelude::*,hw}; const PSTATE_TABLE_ENTRIES: usize = 16;\n"
+    code += item(init, "pub(crate) struct PStateTables {") + "\n"
+    code += item(init, "impl PStateTables {") + "\n}\n"
+    code += "mod m3_mmio_26 { pub const SLOTS:usize=53; pub const RECORD_BYTES:usize=40; }\n"
+    code += f'#[path = "{M3}/m3_power_26.rs"] mod m3_power_26;\n'
+    code += fixture
+    for path in [M3 / "hw.rs", M3 / "initdata.rs", M3 / "m3_power_26.rs", ROOT / "drivers/gpu/drm/asahi/float.rs"]:
+        print(hashlib.sha256(path.read_bytes()).hexdigest(), path.relative_to(ROOT), flush=True)
+    with tempfile.TemporaryDirectory(prefix="m3-power-") as directory:
+        source = Path(directory) / "test.rs"
+        source.write_text(code)
+        for flags in [[], ["-O"]]:
+            binary = Path(directory) / "tests"
+            subprocess.run(["rustc", "--edition=2021", "--test", *flags, str(source), "-o", str(binary)], check=True)
+            subprocess.run([str(binary)], check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/asahi/test-m3-uapi.py b/tools/asahi/test-m3-uapi.py
new file mode 100755
index 0000000000000..a63249e18869d
--- /dev/null
+++ b/tools/asahi/test-m3-uapi.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Compile public C producers and run the production Rust parser offline."""
+import argparse
+import hashlib
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+
+
+def run(command, **kwargs):
+    subprocess.run(command, check=True, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--mesa", type=Path, help="matched public Mesa source")
+    args = parser.parse_args()
+    root = Path(__file__).resolve().parents[2]
+    if sys.byteorder != "little":
+        raise SystemExit("C fixtures currently require a little-endian host")
+    cc = os.environ.get("CC", "cc")
+    rustc = os.environ.get("RUSTC", "rustc")
+    fixture_source = root / "tools/asahi/m3-uapi-fixture.c"
+    with tempfile.TemporaryDirectory(prefix="asahi-uapi-") as temporary:
+        work = Path(temporary)
+        headers = work / "headers"
+        run(["make", "-s", "-C", str(root), "O=" + str(work / "headers-build"),
+             "ARCH=arm64", "headers_install", "INSTALL_HDR_PATH=" + str(headers)])
+        sources = [(headers / "include", "drm/asahi_drm.h")]
+        if args.mesa:
+            sources.append((args.mesa.resolve() / "include", "drm-uapi/asahi_drm.h"))
+        fixtures = []
+        layouts = []
+        prefixes = []
+        for index, (includes, header) in enumerate(sources):
+            producer = work / ("producer-" + str(index))
+            run([cc, "-std=c11", "-Wall", "-Wextra", "-Werror",
+                 "-I" + str(includes), '-DASAHI_HEADER="' + header + '"',
+                 str(fixture_source), "-o", str(producer)])
+            prefix = work / ("prefix-" + str(index))
+            with prefix.open("wb") as output:
+                run([str(producer)], stdout=output)
+            prefixes.append(prefix)
+            layout = work / ("layout-" + str(index))
+            with layout.open("wb") as output:
+                run([str(producer), "--layout"], stdout=output)
+            layouts.append(layout)
+            fixture = work / ("fixture-" + str(index))
+            with fixture.open("wb") as output:
+                run([str(producer), "--full"], stdout=output)
+            fixtures.append(fixture)
+        reference = prefixes[0].read_bytes()
+        if any(prefix.read_bytes() != reference for prefix in prefixes[1:]):
+            raise SystemExit("kernel/Mesa canonical fixture mismatch")
+        print("canonical fixture sha256=" + hashlib.sha256(reference).hexdigest(), flush=True)
+        for optimized in (False, True):
+            binary = work / ("tests-optimized" if optimized else "tests-debug")
+            command = [rustc, "--edition=2021", "--test",
+                       str(root / "tools/asahi/m3-uapi-tests.rs"), "-o", str(binary)]
+            if optimized:
+                command.append("-O")
+            run(command)
+            for index, fixture in enumerate(prefixes + fixtures):
+                env = dict(os.environ, ASAHI_UAPI_FIXTURE=str(fixture),
+                           ASAHI_UAPI_LAYOUT=str(layouts[index % len(layouts)]))
+                run([str(binary)], env=env)
+
+
+if __name__ == "__main__":
+    main()