Fix M3 Pro GPU bandwidth and G15 submission overhead

drivers/gpu/drm/asahi/g15/boot.rs

@@ -73,6 +73,7 @@ pub(crate) mod glue {
         fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool;
         pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int;
         pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int;
+        pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int;
         pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void);
         fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool;
     }

drivers/gpu/drm/asahi/g15/command.rs

@@ -146,6 +146,11 @@ pub(crate) struct Context<'a> {
     pub(crate) usc_base: u64,
 }
 
+struct CheckedContext<'a> {
+    vm: &'a mut mmu::VmReadGuard,
+    usc_base: u64,
+}
+
 /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are
 /// checked ([`Prepared::validate`]).
 pub(crate) struct Prepared<'a> {
@@ -376,7 +381,13 @@ impl Timestamps<'_> {
 /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested
 /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a
 /// page without the access EACCES.
-pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result {
+pub(crate) fn cover(
+    vm: &mut mmu::VmReadGuard,
+    start: u64,
+    end: u64,
+    read: bool,
+    write: bool,
+) -> Result {
     let mask = mmu::UAT_PGMSK as u64;
     if end <= start {
         return Err(EINVAL);
@@ -406,7 +417,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool)
 }
 
 /// Attachment spans, 128-byte granular, actually mapped with `write` or read access.
-fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result {
+fn check_attachments(
+    vm: &mut mmu::VmReadGuard,
+    list: &[uapi::drm_asahi_attachment],
+    write: bool,
+) -> Result {
     for attachment in list {
         let end = attachment
             .pointer
@@ -466,8 +481,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) {
 }
 
 /// Per-command render admission.
-fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result {
-    let vm = context.vm;
+fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result {
+    let vm = &mut *context.vm;
     let allowed = PROCESS_EMPTY_TILES
         | NO_VERTEX_CLUSTERING
         | RSRC_SPEC_HI
@@ -551,11 +566,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu
 
 /// Per-command compute admission.
 fn check_compute(
-    context: &Context<'_>,
+    context: &mut CheckedContext<'_>,
     cmd: &uapi::drm_asahi_cmd_compute,
     attachments: &[uapi::drm_asahi_attachment],
 ) -> Result {
-    let vm = context.vm;
+    let vm = &mut *context.vm;
     let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end);
     if cmd.flags != 0
         || cmd.sampler_count != 0
@@ -649,6 +664,12 @@ impl Prepared<'_> {
             counts,
             timestamps,
         } = self;
+        // On error, unlock before the last timestamp mapping can drop and unmap.
+        let mut access = context.vm.read_lock()?;
+        let mut context = CheckedContext {
+            vm: &mut access,
+            usc_base: context.usc_base,
+        };
         let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?;
         let mut compute = KVec::new();
         let mut index = 0;
@@ -657,12 +678,12 @@ impl Prepared<'_> {
             match entry.kind {
                 SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => {
                     // Validated readable, never serialized on G15.
-                    check_attachments(context.vm, &attachment_list(entry.payload)?, false)?;
+                    check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?;
                 }
                 SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?,
                 CMD_RENDER => {
                     let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?;
-                    check_render(context, &cmd)?;
+                    check_render(&mut context, &cmd)?;
                     commands.push(
                         Command::Render(KBox::new(
                             RenderCommand {
@@ -677,7 +698,7 @@ impl Prepared<'_> {
                 }
                 CMD_COMPUTE => {
                     let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?;
-                    check_compute(context, &cmd, &compute)?;
+                    check_compute(&mut context, &cmd, &compute)?;
                     let slots = resolved.get(index).ok_or(EINVAL)?;
                     commands.push(
                         Command::Compute(KBox::new(

drivers/gpu/drm/asahi/g15/gate.rs

@@ -71,7 +71,7 @@ struct Window {
 impl Window {
     /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page
     /// unless the record crosses it). Unreadable memory: EFAULT.
-    fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> {
+    fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> {
         let n = count * 4;
         let cached = self.len != 0
             && va >= self.va
@@ -108,7 +108,7 @@ struct Checked {
 
 /// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is
 /// spent.
-fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result {
+fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result {
     if *count >= CDM_MAX_PAGE_CHECKS {
         return Err(ENOSPC);
     }
@@ -118,7 +118,7 @@ fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result {
 
 /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable.
 /// The last proven program page is not checked again.
-fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result {
+fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result {
     if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 {
         return Err(EINVAL);
     }
@@ -134,7 +134,7 @@ fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Resul
 
 /// An indirect launch's 12 argument bytes: readable. The last proven argument page is not
 /// checked again; arguments that cross a page are checked each time.
-fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result {
+fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result {
     let end = address.checked_add(12).ok_or(EINVAL)?;
     let page = address & !(mmu::UAT_PGMSK as u64);
     let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page;
@@ -149,7 +149,7 @@ fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result {
 }
 
 /// VDM admission: a normal root stream opens with exactly the G15 barrier.
-pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result {
+pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result {
     if !params::stream_gate() {
         return Ok(());
     }
@@ -165,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result {
 /// CDM admission: walks the stream from its base through no-return links until TERMINATE,
 /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers
 /// (4), link (8).
-pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result {
+pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result {
     if !params::stream_gate() {
         return Ok(());
     }

drivers/gpu/drm/asahi/g15/glue.c

@@ -35,6 +35,7 @@ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size);
 bool asahi_g15_dma_coherent(struct device *dev);
 int asahi_g15_pmp_prepare(struct device *dev);
 int asahi_g15_pmp_get(struct device *dev);
+int asahi_g15_pmp_ready(void);
 void asahi_g15_pmp_put(struct device *dev);
 bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size);
 
@@ -150,6 +151,11 @@ int asahi_g15_pmp_get(struct device *dev)
 
 }
 
+int asahi_g15_pmp_ready(void)
+{
+	return apple_pmp_t6030_gpu_ready();
+}
+
 /*
  * Give back the vote of asahi_g15_pmp_get() after a start that failed before
  * the firmware was touched. The full M3 stack's J514S bridge has no release:

drivers/gpu/drm/asahi/g15/mod.rs

@@ -539,7 +539,21 @@ impl Firmware {
                 vms,
             },
         );
-        if let Err(e) = firmware.init_graphs() {
+        let result = firmware.init_graphs().and_then(|_| {
+            if cfg.platform != hw::Platform::AdtWithPmp {
+                return Ok(());
+            }
+            // A report before firmware startup is ACKed without enabling bandwidth scaling.
+            // Reassert it after setup, including the backend's final device control.
+            // SAFETY: This session retains its acknowledged T6030 AGX power vote.
+            let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };
+            if ret == 0 {
+                Ok(())
+            } else {
+                Err(Error::from_errno(ret))
+            }
+        });
+        if let Err(e) = result {
             let Firmware {
                 session,
                 init,

drivers/gpu/drm/asahi/mmu.rs

@@ -358,6 +358,20 @@ fn lock_vm(
     Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? }))
 }
 
+#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)]
+pub(crate) struct VmReadGuard(VmExecGuard);
+
+#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)]
+impl VmReadGuard {
+    pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool {
+        Vm::covers_range_locked(&mut self.0, address, size, read, write)
+    }
+
+    pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result {
+        Vm::read_user_locked(&mut self.0, va, buf)
+    }
+}
+
 /// Slot binding-related inner data for a Vm instance.
 struct VmBinding {
     active_users: usize,
@@ -1804,7 +1818,16 @@ impl Vm {
     /// mappings cannot change during the copy (GPUVM exec lock).
     #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)]
     pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result {
-        let mut inner = lock_vm(&self.gpuvm, None, false)?;
+        self.read_lock()?.read_user(va, buf)
+    }
+
+    #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)]
+    pub(crate) fn read_lock(&self) -> Result<VmReadGuard> {
+        Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?))
+    }
+
+    #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)]
+    fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result {
         let mut done = 0usize;
         while done < buf.len() {
             let address = va.checked_add(done as u64).ok_or(EFAULT)?;

drivers/pmdomain/apple/pmp-report.c

@@ -1292,6 +1292,62 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id,
 		50000);
 }
 
+/* Find and check the report before taking the GPU vote; optionally update it. */
+static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable)
+{
+	struct apple_pmp_report *rep, *found = NULL;
+	struct device_node *pmp;
+	const u8 *table;
+	int len = 0, ret = -ENODEV;
+
+	if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) ||
+	    !of_machine_is_compatible("apple,t6030") ||
+	    !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2"))
+		return -ENODEV;
+
+	/* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */
+	table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len);
+	if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) ||
+	    memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8))
+		return -EINVAL;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(rep, &apple_pmp_reports, list) {
+		if (rep->offsets != &apple_pmp_offsets_t6030)
+			continue;
+		pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0);
+		if (pmp == supplier->of_node) {
+			of_node_put(pmp);
+			if (found)
+				goto out;
+			found = rep;
+		} else {
+			of_node_put(pmp);
+		}
+	}
+	if (found) {
+		if (readq(found->base + found->offsets->status) != PMP_REPORT_READY)
+			ret = -EAGAIN;
+		else
+			ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0;
+	}
+out:
+	rcu_read_unlock();
+	return ret;
+}
+int apple_pmp_report_check_t6030_gfx(struct device *supplier)
+{
+	return apple_pmp_report_t6030_gfx(supplier, NULL);
+}
+EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx);
+
+/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */
+int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable)
+{
+	return apple_pmp_report_t6030_gfx(supplier, &enable);
+}
+EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx);
+
 static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable)
 {
 	struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd);

drivers/soc/apple/pmp-export.c

@@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device);
 
 int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled)
 {
+	bool report_gfx;
 	int ret;
 
 	if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1)
@@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled)
 	ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000);
 	if (ret)
 		return ret;
-	return apple_pmp_send_power_command(pmp_data, command, device_id, enabled);
+	/* T6030 needs both the RTKit power vote and the PTD bandwidth report. */
+	report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) &&
+		     command == 0x0f && device_id == 5 &&
+		     of_machine_is_compatible("apple,t6030") &&
+		     of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2");
+	if (report_gfx) {
+		ret = apple_pmp_report_check_t6030_gfx(pmp_device);
+		if (ret)
+			return ret;
+	}
+	if (report_gfx && !enabled) {
+		ret = apple_pmp_report_set_t6030_gfx(pmp_device, false);
+		if (ret) {
+			/* Clearing was written before the ACK timed out. Restore it. */
+			apple_pmp_report_set_t6030_gfx(pmp_device, true);
+			return ret;
+		}
+	}
+	ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled);
+	/* Power is unknown after a lost reply; assert the report conservatively. */
+	if (report_gfx && !enabled && ret)
+		apple_pmp_report_set_t6030_gfx(pmp_device, true);
+	return ret;
 }
 EXPORT_SYMBOL_GPL(apple_pmp_set_device_power);
+
+/* Reassert after startup, including a report left set by an earlier failure. */
+int apple_pmp_t6030_gpu_ready(void)
+{
+	int ret;
+
+	guard(mutex)(&pmp_lock);
+	if (!pmp_data)
+		return -ENODEV;
+	ret = apple_pmp_report_set_t6030_gfx(pmp_device, false);
+	if (ret)
+		return ret;
+	return apple_pmp_report_set_t6030_gfx(pmp_device, true);
+}
+EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready);

include/linux/soc/apple/pmp-report.h

@@ -15,6 +15,9 @@ struct device;
 
 #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT)
 int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms);
+/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */
+int apple_pmp_report_check_t6030_gfx(struct device *supplier);
+int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable);
 int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout);
 int apple_pmp_report_fast_die_effort(const struct device_node *np,
 				     unsigned int lane, u8 *effort);
@@ -25,6 +28,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier,
 	return -ENODEV;
 }
 
+static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier)
+{
+	return -ENODEV;
+}
+
+static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable)
+{
+	return -ENODEV;
+}
+
 static inline int apple_pmp_report_wait_ready(struct device_node *entry,
 					      unsigned long timeout)
 {