Merge integration/aurora into aurora-wip

.github/workflows/sep-source.yml

@@ -3,6 +3,9 @@ name: SEP source checks
 on:
   pull_request:
     paths:
+      - 'Makefile'
+      - 'rust/**'
+      - 'tools/aurora-bindgen/**'
       - 'drivers/soc/apple/**'
       - 'include/uapi/linux/apple-sep.h'
       - 'tools/aurora-sep/**'

drivers/gpu/drm/asahi/g17/completion.rs

@@ -631,6 +631,11 @@ impl Completion {
         drop(lease);
     }
 
+    /// Whether the command still holds its VM pin: firmware may still own or replay it.
+    pub(crate) fn retains_vm(&self) -> bool {
+        self.vm_job.lock().is_some()
+    }
+
     fn release_healthy(&self) {
         if let Some(sources) = &self.sources {
             sources.release();

drivers/gpu/drm/asahi/g17/dependency.rs

@@ -25,6 +25,48 @@ pub(crate) fn replay_status(status: i32) -> Result {
     else { Ok(()) }
 }
 
+/// Whether a failed command may still run or write: it holds its VM pin until it has physically
+/// retired, or was never published.
+pub(crate) trait Retention {
+    fn retained(&self) -> bool;
+}
+
+/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix
+/// covers one is cancelled, so none runs before a failed producer the firmware may still replay.
+/// Once that producer has retired nothing of it can run or write any more, so it stops counting:
+/// otherwise one cancelled command cancelled every later command of its queue for good.
+pub(crate) struct FailureRecord<T> {
+    retained: KVec<(u64, T)>,
+    /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did.
+    untracked: Option<(u64, T)>,
+}
+
+impl<T: Retention + Clone> FailureRecord<T> {
+    pub(crate) const fn new() -> Self {
+        Self { retained: KVec::new(), untracked: None }
+    }
+
+    /// A finished frontier entry, at pruning.
+    pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) {
+        if status >= 0 || !entry.retained() {
+            return;
+        }
+        if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err()
+            && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first)
+        {
+            self.untracked = Some((sequence, entry.clone()));
+        }
+    }
+
+    /// The earliest failure later prefixes must not bypass.
+    pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> {
+        self.retained.retain(|(_, entry)| entry.retained());
+        Iterator::chain(self.retained.iter(), self.untracked.iter())
+            .min_by_key(|(sequence, _)| *sequence)
+            .map(|(sequence, entry)| (*sequence, entry))
+    }
+}
+
 pub(crate) struct Frontier {
     dependencies: [KickDependency; KICK_BARRIERS_MAX],
     sequences: [u64; KICK_BARRIERS_MAX],
@@ -89,3 +131,68 @@ impl Frontier {
         self.insert(sequence, dependency)
     }
 }
+
+#[kunit_tests(asahi_g17_dependency)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct Probe(kernel::sync::Arc<core::sync::atomic::AtomicBool>);
+
+    impl Probe {
+        fn new(retained: bool) -> Self {
+            Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap())
+        }
+        fn retire(&self) {
+            self.0.store(false, core::sync::atomic::Ordering::Relaxed);
+        }
+    }
+
+    impl Retention for Probe {
+        fn retained(&self) -> bool {
+            self.0.load(core::sync::atomic::Ordering::Relaxed)
+        }
+    }
+
+    fn earliest(record: &mut FailureRecord<Probe>) -> Option<u64> {
+        record.earliest().map(|(sequence, _)| sequence)
+    }
+
+    #[test]
+    fn a_cancelled_command_does_not_poison_its_queue() {
+        // Cancelled or failed before publication: it released its pin and can never run.
+        let mut record = FailureRecord::new();
+        record.note(5, ECANCELED.to_errno(), &Probe::new(false));
+        assert_eq!(earliest(&mut record), None);
+    }
+
+    #[test]
+    fn a_retained_failure_cancels_until_it_retires() {
+        let mut record = FailureRecord::new();
+        let failed = Probe::new(true);
+        record.note(5, EIO.to_errno(), &failed);
+        assert_eq!(earliest(&mut record), Some(5));
+        failed.retire();
+        assert_eq!(earliest(&mut record), None);
+    }
+
+    #[test]
+    fn a_later_retained_failure_survives_an_earlier_retirement() {
+        let mut record = FailureRecord::new();
+        let (first, second) = (Probe::new(true), Probe::new(true));
+        record.note(5, EIO.to_errno(), &first);
+        record.note(9, ECANCELED.to_errno(), &second);
+        assert_eq!(earliest(&mut record), Some(5));
+        first.retire();
+        assert_eq!(earliest(&mut record), Some(9));
+        second.retire();
+        assert_eq!(earliest(&mut record), None);
+    }
+
+    #[test]
+    fn successes_are_not_recorded() {
+        let mut record = FailureRecord::new();
+        record.note(5, 1, &Probe::new(true));
+        assert_eq!(earliest(&mut record), None);
+    }
+}

drivers/gpu/drm/asahi/g17/job.rs

@@ -401,6 +401,13 @@ pub(crate) struct Fences {
     pub(crate) ready: Fence,
     pub(crate) completed: Fence,
     pub(crate) publication: Option<Arc<super::completion::Publication>>,
+    pub(crate) owner: Arc<Completion>,
+}
+
+impl super::dependency::Retention for Fences {
+    fn retained(&self) -> bool {
+        self.owner.retains_vm()
+    }
 }
 
 impl Fences {
@@ -479,6 +486,6 @@ impl<B: Backend> Scheduler<B> {
             self.backend.count_backlog(&packet);
         }
         job.push();
-        Ok(Fences { ready, completed, publication: host_publication })
+        Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() })
     }
 }

drivers/gpu/drm/asahi/g17/queue/mod.rs

@@ -10,6 +10,7 @@ use super::{
     command::{Parser, UscWindow, Validated},
     completion::{Completion, Destinations, PreparedPublication},
     context::Context,
+    dependency::FailureRecord,
     fence::Outputs,
     job::{self, Backend, Engine, Fences, Order, Packet, Scheduler},
     validation::Proof,
@@ -99,7 +100,7 @@ pub(crate) struct Queue<B: Backend> {
     /// Finished frontier entries pruned during this ioctl. Their last references
     /// can tear a whole completion down; that waits until the ioctl has published.
     finished: KVec<Fences>,
-    failures: [Option<(u64, Fence)>; 2],
+    failures: [FailureRecord<Fences>; 2],
     // Closing the userspace queue releases admission even if firmware still
     // retains one of its execution contexts or installed physical graphs.
     _slot: admission::QueueSlot,
@@ -143,7 +144,7 @@ impl<B: Backend> Queue<B> {
             sequence: [0; 2],
             frontiers: core::array::from_fn(|_| KVec::new()),
             finished: KVec::new(),
-            failures: [None, None],
+            failures: [FailureRecord::new(), FailureRecord::new()],
             _slot: slot,
         })
     }
@@ -182,11 +183,7 @@ impl<B: Backend> Queue<B> {
             if let Some(finished) = finished.as_mut() {
                 let _ = finished.push_within_capacity(fences.clone());
             }
-            // One earliest failed ordinal covers every later prefix without
-            // retaining its packet, mappings, or every failed fence.
-            if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) {
-                *failure = Some((*sequence, fences.completed.clone()));
-            }
+            failure.note(*sequence, status, fences);
             false
         });
     }
@@ -242,9 +239,9 @@ impl<B: Backend> Queue<B> {
                     || fences.ready.raw() != fences.completed.raw()
                     || fences.status() > 0
             })
-            && self.failures.iter().enumerate().all(|(index, failure)| {
-                failure.as_ref().is_none_or(|(sequence, _)| {
-                    !Order::contains(order.wait_through[index], *sequence)
+            && self.failures.iter_mut().enumerate().all(|(index, failure)| {
+                failure.earliest().is_none_or(|(sequence, _)| {
+                    !Order::contains(order.wait_through[index], sequence)
                 })
             }) {
             self.backend.input_sources(inputs)
@@ -314,11 +311,12 @@ impl<B: Backend> Queue<B> {
             .iter()
             .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw())
             .count();
-        let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| {
-            self.failures[index].as_ref().and_then(|(sequence, fence)| {
-                Order::contains(order.wait_through[index], *sequence).then_some(fence)
-            })
-        });
+        let mut failures: [Option<Fence>; 2] = [None, None];
+        for (index, failure) in self.failures.iter_mut().enumerate() {
+            failures[index] = failure.earliest().and_then(|(sequence, fences)| {
+                Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone())
+            });
+        }
         let failure_count = failures.iter().flatten().count();
         let mut dependencies = KVec::with_capacity(
             inputs.len() + count + timestamp_count + failure_count,

drivers/gpu/drm/asahi/mmu.rs

@@ -1307,9 +1307,7 @@ impl KernelMapping {
             return Err(EINVAL);
         }
         let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?;
-        lock_vm(&self.0.owner, None, false)?
-            .page_table
-            .translate_iova(address)
+        self.0.owner.lock_inner().page_table.translate_iova(address)
     }
 
     /// Writes a userspace timestamp through the retained GEM mapping. Aligned results
@@ -2263,9 +2261,7 @@ impl Vm {
     }
 
     pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool {
-        let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else {
-            return false;
-        };
+        let mut inner = self.gpuvm.lock_inner();
         Self::covers_range_locked(&mut inner, address, size, read, write)
     }
 
@@ -2280,9 +2276,7 @@ impl Vm {
         if stride <= block_size {
             return false;
         }
-        let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else {
-            return false;
-        };
+        let mut inner = self.gpuvm.lock_inner();
         addresses.iter().all(|&address| {
             let Some(guard) = address.checked_add(block_size) else {
                 return false;
@@ -2292,12 +2286,12 @@ impl Vm {
         })
     }
 
-    /// Checks all exact request tuples under one GPUVM exec lock. Access bit0
+    /// Checks all exact request tuples under one hold of the VM reservation. Access bit0
     /// requests reading and bit1 writing. The epoch names this same snapshot.
     pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option<u64>) {
-        let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else {
-            return (false, None);
-        };
+        // Every mapping change holds the VM reservation, so readers need only that one. The full
+        // exec lock would also lock every external BO, making each submit O(external BOs).
+        let mut inner = self.gpuvm.lock_inner();
         let epoch = inner.epoch;
         let covered = ranges.iter().all(|&(address, size, access)| {
             Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0)
@@ -2421,7 +2415,7 @@ impl Vm {
         // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm;
         // every subsequent access to VM data is serialized by the exec lock.
         let mut vm_exec = unsafe {
-            self.gpuvm.exec_lock(/*Some(gem)*/ None, false)?
+            self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)?
         };
         let inner = vm_exec.unique_gpuvm.data();
 
@@ -2508,7 +2502,7 @@ impl Vm {
         // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe
         let mut vm_exec = unsafe {
             self.gpuvm
-                .exec_lock(/*Some(&gem)*/ None, false)
+                .exec_lock_vm(/*Some(&gem)*/ None, false)
                 .inspect_err(|_| {
                     dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n");
                 })?
@@ -2598,7 +2592,7 @@ impl Vm {
         // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm;
         // every subsequent access to VM data is serialized by the exec lock.
         let mut vm_exec = unsafe {
-            self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)?
+            self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)?
         };
         let inner = vm_exec.unique_gpuvm.data();
 
@@ -2671,7 +2665,7 @@ impl Vm {
     ) -> Result<KernelMapping> {
         // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm;
         // every subsequent access to VM data is serialized by the exec lock.
-        let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? };
+        let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? };
 
         if (iova as usize | phys | size) & UAT_PGMSK != 0 {
             dev_err!(
@@ -2777,7 +2771,7 @@ impl Vm {
 
         // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm;
         // every subsequent access to VM data is serialized by the exec lock.
-        let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? };
+        let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? };
 
         mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size);
         let _commit = if user {
@@ -2808,7 +2802,7 @@ impl Vm {
         let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?;
 
         // SAFETY: See `lock_vm()`.
-        let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } {
+        let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } {
             Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem),
             result => result?,
         };
@@ -3187,7 +3181,7 @@ impl Uat {
             return Err(EINVAL);
         }
         let root = {
-            let owner = lock_vm(&vm.gpuvm, None, false)?;
+            let owner = vm.gpuvm.lock_inner();
             if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) {
                 return Err(EINVAL);
             }
@@ -3332,7 +3326,7 @@ impl Uat {
             return Err(EINVAL);
         }
         {
-            let owner = lock_vm(&vm.gpuvm, None, false)?;
+            let owner = vm.gpuvm.lock_inner();
             if owner.is_kernel
                 || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner))
             {
@@ -3727,7 +3721,7 @@ impl Vm {
     /// Diagnostic reservation occupancy (including guards), not a placement promise.
     /// Removal can progress between queries; allocations are excluded by the VM lock.
     pub(crate) fn capacity(&self, range: Range<u64>) -> Result<(u64, u64)> {
-        let inner = lock_vm(&self.gpuvm, None, false)?;
+        let inner = self.gpuvm.lock_inner();
         let mut next = range.start;
         let mut used = 0;
         let mut largest = 0;

drivers/gpu/drm/asahi/mmu/bind.rs

@@ -132,7 +132,7 @@ impl Vm {
                 PreparedUserBindOp::Map(map) => {
                     // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is
                     // not locked: the map step may still populate its pages.
-                    let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? };
+                    let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? };
                     match self.mapping_commit() {
                         Ok(commit) => {
                             let (flags, repeat) = if map.single_page {
@@ -177,7 +177,7 @@ impl Vm {
                 }
                 PreparedUserBindOp::Unmap(unmap) => {
                     // SAFETY: See `lock_vm()`.
-                    let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? };
+                    let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? };
                     match self.mapping_commit() {
                         Ok(commit) => {
                             let mut ctx =