drm/asahi: carry G15 submission fixes and T6030 PMP votes with host coverage

drivers/gpu/drm/asahi/g15/boot.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Admission and firmware start to READY.
@@ -73,6 +73,7 @@
         fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool;
         pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int;
         pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int;
+        pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int;
         pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void);
         fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool;
     }

drivers/gpu/drm/asahi/g15/command.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! SUBMIT parsing and validation into engine commands.
@@ -146,6 +146,11 @@
     pub(crate) usc_base: u64,
 }
 
+struct CheckedContext<'a> {
+    vm: &'a mut mmu::VmReadGuard,
+    usc_base: u64,
+}
+
 /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are
 /// checked ([`Prepared::validate`]).
 pub(crate) struct Prepared<'a> {
@@ -376,7 +381,13 @@
 /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested
 /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a
 /// page without the access EACCES.
-pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result {
+pub(crate) fn cover(
+    vm: &mut mmu::VmReadGuard,
+    start: u64,
+    end: u64,
+    read: bool,
+    write: bool,
+) -> Result {
     let mask = mmu::UAT_PGMSK as u64;
     if end <= start {
         return Err(EINVAL);
@@ -406,7 +417,11 @@
 }
 
 /// Attachment spans, 128-byte granular, actually mapped with `write` or read access.
-fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result {
+fn check_attachments(
+    vm: &mut mmu::VmReadGuard,
+    list: &[uapi::drm_asahi_attachment],
+    write: bool,
+) -> Result {
     for attachment in list {
         let end = attachment
             .pointer
@@ -466,8 +481,8 @@
 }
 
 /// Per-command render admission.
-fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result {
-    let vm = context.vm;
+fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result {
+    let vm = &mut *context.vm;
     let allowed = PROCESS_EMPTY_TILES
         | NO_VERTEX_CLUSTERING
         | RSRC_SPEC_HI
@@ -551,11 +566,11 @@
 
 /// Per-command compute admission.
 fn check_compute(
-    context: &Context<'_>,
+    context: &mut CheckedContext<'_>,
     cmd: &uapi::drm_asahi_cmd_compute,
     attachments: &[uapi::drm_asahi_attachment],
 ) -> Result {
-    let vm = context.vm;
+    let vm = &mut *context.vm;
     let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end);
     if cmd.flags != 0
         || cmd.sampler_count != 0
@@ -649,6 +664,12 @@
             counts,
             timestamps,
         } = self;
+        // On error, unlock before the last timestamp mapping can drop and unmap.
+        let mut access = context.vm.read_lock()?;
+        let mut context = CheckedContext {
+            vm: &mut access,
+            usc_base: context.usc_base,
+        };
         let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?;
         let mut compute = KVec::new();
         let mut index = 0;
@@ -657,12 +678,12 @@
             match entry.kind {
                 SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => {
                     // Validated readable, never serialized on G15.
-                    check_attachments(context.vm, &attachment_list(entry.payload)?, false)?;
+                    check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?;
                 }
                 SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?,
                 CMD_RENDER => {
                     let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?;
-                    check_render(context, &cmd)?;
+                    check_render(&mut context, &cmd)?;
                     commands.push(
                         Command::Render(KBox::new(
                             RenderCommand {
@@ -677,7 +698,7 @@
                 }
                 CMD_COMPUTE => {
                     let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?;
-                    check_compute(context, &cmd, &compute)?;
+                    check_compute(&mut context, &cmd, &compute)?;
                     let slots = resolved.get(index).ok_or(EINVAL)?;
                     commands.push(
                         Command::Compute(KBox::new(

drivers/gpu/drm/asahi/g15/encode.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Work-image and microsequence encoders.
@@ -64,10 +64,9 @@
 }
 
 /// Firmware stamp value of ordinal `n` on the queue with this seed.
-/// Stamps step by 0x100 through 0xffffff values and never read 0.
+/// Stamps step by 0x100, including zero when the counter wraps.
 pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 {
-    let base = (seed / 0x100) as u64;
-    ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32
+    seed.wrapping_add((n as u32).wrapping_mul(0x100))
 }
 
 /// Compute stamp of ordinal `n`.

drivers/gpu/drm/asahi/g15/engine.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The prefix/horizon/flight engine.
@@ -155,10 +155,10 @@
     pub(crate) fs_start: u32,
     /// Channel producers (TA, FS, compute) after the prefix's records.
     pub(crate) rec: [u32; 3],
-    /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none):
+    /// Compute member j's FW stamp target, and the FS target it waits on directly:
     /// its predecessors in the prefix (for the compute-hang check).
     pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS],
-    pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS],
+    pub(crate) cwait: [Option<u32>; sizes::COMPUTE_SLOTS],
     /// Compute slots, pass slots and clustering buffers it holds.
     pub(crate) claims: Claims,
     /// The prefix rebuilds the render allocators (priming); nothing is published behind it.
@@ -189,7 +189,7 @@
             fs_start: 0,
             rec: [0; 3],
             cstamp: [0; sizes::COMPUTE_SLOTS],
-            cwait: [0; sizes::COMPUTE_SLOTS],
+            cwait: [None; sizes::COMPUTE_SLOTS],
             claims: Claims::new(),
             rebuild: false,
             doomed: false,
@@ -385,9 +385,11 @@
     pub(crate) rebuild_live: bool,
     /// A ready prefix waits only for a record's consumption: repoll sooner.
     pub(crate) chan_wait: bool,
+    /// A ready job waits for an ASID held by a completing failed job.
+    pub(crate) slot_wait: bool,
     /// Progress tracking of the oldest prefix.
     pub(crate) progress_flight: u64,
-    pub(crate) progress: [u32; 3],
+    pub(crate) progress: [Option<u32>; 3],
     pub(crate) stats: Stats,
     /// Phase timers (debug_flags bit 49).
     pub(crate) prof: Prof,
@@ -424,8 +426,9 @@
             render_registered: false,
             rebuild_live: false,
             chan_wait: false,
+            slot_wait: false,
             progress_flight: 0,
-            progress: [0; 3],
+            progress: [None; 3],
             stats: Stats::default(),
             prof: Prof::default(),
             debugfs: None,
@@ -546,6 +549,7 @@
         let mut result = Ok(());
 
         self.engine.chan_wait = false;
+        self.engine.slot_wait = false;
         // Nothing is published while a hang request is armed.
         while self.recovery.failed().is_none()
             && !self.engine.frozen
@@ -592,7 +596,7 @@
                         dev_err!(
                             self.graph.dev.as_ref(),
                             "G15: publication of ctx {} failed after a queue WRITE ({:?})\n",
-                            job.vm.slot,
+                            job.slot(),
                             e
                         );
                         complete(done, &job, Err(EIO));
@@ -675,14 +679,27 @@
     /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE
     /// advanced; any other error means nothing was published.
     fn publish_prefix(&mut self, job: &Arc<Job>, done: &mut Completions) -> Result {
+        if let Err(error) = job.bind() {
+            if error == ENOSPC {
+                // A failed job can keep its binding until scheduler references drop after
+                // this pass. Leave healthy work pending and retry without the device lock.
+                self.engine.slot_wait = true;
+                return Err(EAGAIN);
+            }
+            return Err(error);
+        }
         let mut f = Flight::new(job.clone());
         let result = self.publish_prefix_into(job, &mut f);
         match result {
             Err(e) if e != EIO => {
                 // Never published: its storage is free again, and horizon jobs taken into it
                 // fail with it.
                 self.slots.release(&mut f.claims);
+                if job.progress.next() == job.progress.sent() {
+                    job.release_binding();
+                }
                 for j2 in f.group.iter() {
+                    j2.release_binding();
                     complete(done, j2, Err(e));
                 }
             }
@@ -899,11 +916,22 @@
                     hz_stop = Some(1);
                     break;
                 }
-                for k in 0..n2 {
-                    match command(&j2, k).ok_or(EINVAL)? {
-                        Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?,
-                        Command::Render(_) => return Err(EINVAL),
+                j2.bind()?;
+                let encoded = (|| {
+                    for k in 0..n2 {
+                        match command(&j2, k).ok_or(EINVAL)? {
+                            Command::Compute(c) => {
+                                self.prefix_compute(&j2, c, &mut pf, f, false)?
+                            }
+                            Command::Render(_) => return Err(EINVAL),
+                        }
                     }
+                    Ok(())
+                })();
+                if let Err(error) = encoded {
+                    // Not in the horizon yet and nothing of it reached the firmware.
+                    j2.release_binding();
+                    return Err(error);
                 }
                 computes += n2;
                 j2.progress.set_sent(n2);
@@ -1057,7 +1085,7 @@
             Queue,
             self.graph.dev,
             "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n",
-            job.vm.slot,
+            job.slot(),
             computes,
             pf.n,
             pf.c_head,
@@ -1095,7 +1123,7 @@
     ) -> Result {
         let n = pf.n + 1;
         let t = encode::stamp(n);
-        let ctx = job.vm.slot;
+        let ctx = job.slot();
         let ts = &cmd.timestamps;
         let mut lap = Lap::start();
         // Nothing published yet on failure.
@@ -1191,11 +1219,7 @@
         }
         let j = pf.computes as usize;
         f.cstamp[j] = t;
-        f.cwait[j] = if pf.prev == Prev::Render {
-            pf.last_fs
-        } else {
-            0
-        };
+        f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs);
         g.ring.w64(8 * pf.c_head as usize, s.work.va)?;
         pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP;
         lap.lap(&mut self.engine.prof.write_ns);
@@ -1240,7 +1264,7 @@
         let k = f.claims.pass[pf.renders as usize] as usize;
         let n = pf.draw + 1;
         let usc = job.queue.usc_base;
-        let ctx = job.vm.slot;
+        let ctx = job.slot();
         let tta = encode::stamp_seed(ids::TA_SEED, n);
         let tfs = encode::stamp_seed(ids::FS_SEED, n);
         // Early tiling: a later render member after a render of the same prefix whose render

drivers/gpu/drm/asahi/g15/gate.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Stream admission gates (safety nets for userspace encoders).
@@ -71,7 +71,7 @@
 impl Window {
     /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page
     /// unless the record crosses it). Unreadable memory: EFAULT.
-    fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> {
+    fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> {
         let n = count * 4;
         let cached = self.len != 0
             && va >= self.va
@@ -108,7 +108,7 @@
 
 /// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is
 /// spent.
-fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result {
+fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result {
     if *count >= CDM_MAX_PAGE_CHECKS {
         return Err(ENOSPC);
     }
@@ -118,7 +118,7 @@
 
 /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable.
 /// The last proven program page is not checked again.
-fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result {
+fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result {
     if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 {
         return Err(EINVAL);
     }
@@ -134,7 +134,7 @@
 
 /// An indirect launch's 12 argument bytes: readable. The last proven argument page is not
 /// checked again; arguments that cross a page are checked each time.
-fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result {
+fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result {
     let end = address.checked_add(12).ok_or(EINVAL)?;
     let page = address & !(mmu::UAT_PGMSK as u64);
     let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page;
@@ -149,7 +149,7 @@
 }
 
 /// VDM admission: a normal root stream opens with exactly the G15 barrier.
-pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result {
+pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result {
     if !params::stream_gate() {
         return Ok(());
     }
@@ -165,7 +165,7 @@
 /// CDM admission: walks the stream from its base through no-return links until TERMINATE,
 /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers
 /// (4), link (8).
-pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result {
+pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result {
     if !params::stream_gate() {
         return Ok(());
     }

drivers/gpu/drm/asahi/g15/glue.c

@@ -35,6 +35,7 @@ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size);
 bool asahi_g15_dma_coherent(struct device *dev);
 int asahi_g15_pmp_prepare(struct device *dev);
 int asahi_g15_pmp_get(struct device *dev);
+int asahi_g15_pmp_ready(void);
 void asahi_g15_pmp_put(struct device *dev);
 bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size);
 
@@ -150,6 +151,11 @@ int asahi_g15_pmp_get(struct device *dev)
 
 }
 
+int asahi_g15_pmp_ready(void)
+{
+	return apple_pmp_t6030_gpu_ready();
+}
+
 /*
  * Give back the vote of asahi_g15_pmp_get() after a start that failed before
  * the firmware was touched. The full M3 stack's J514S bridge has no release:

drivers/gpu/drm/asahi/g15/hw.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! G15 hardware and firmware-ABI constants (layout values are shared by several modules; change
@@ -514,8 +514,6 @@
     pub(crate) const AHEAD_RECORDS: u32 = 64;
     /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit).
     pub(crate) const MAX_QUEUES: u32 = 58;
-    /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63).
-    pub(crate) const MAX_VMS: u32 = 63;
 }
 
 /// Firmware channels, doorbells, seeds and identifiers.