drm/asahi: g17: keep innocent clients alive across another client's recovery

drivers/gpu/drm/asahi/g17/completion.rs

@@ -694,7 +694,9 @@ impl Completion {
             return;
         }
         self.status.record_if_device_loss(error);
-        if self.accepted.load(Ordering::Acquire) {
+        // As in complete(): a cancellation is not this client's device loss. A guilty VM
+        // already carries its own sticky error.
+        if self.accepted.load(Ordering::Acquire) && error != ECANCELED {
             self.status.report_failure(error);
         }
         self.release_healthy();
@@ -713,7 +715,9 @@ impl Completion {
             guard.quarantine();
         }
         if self.spared() {
-            self.status.report_failure(error);
+            if error != ECANCELED || self.interrupted() {
+                self.status.report_failure(error);
+            }
         } else {
             self.report(error);
         }

drivers/gpu/drm/asahi/g17/completion/publication.rs

@@ -8,7 +8,7 @@
 //! nor waits for another command; the last prerequisite publishes a ready result.
 
 use super::Output;
-use crate::g17::{dependency::inherited_error, status::VmStatus};
+use crate::g17::{dependency::PRODUCER_FAILED, status::VmStatus};
 use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}};
 
 type Links = Option<KBox<Link>>;
@@ -96,7 +96,9 @@ impl Publication {
         if let Err(error) = result {
             // Inherited failure is visible to the client without inventing a
             // permanent admission failure in this command's VM.
-            self.status.report_failure(error);
+            if error != ECANCELED {
+                self.status.report_failure(error);
+            }
         }
         self.output.publish(result);
         let result = result.map(|_| ());
@@ -119,10 +121,8 @@ impl Publication {
         let ready = {
             let mut state = self.state.lock();
             debug_assert!(state.remaining != 0);
-            if let Err(error) = result {
-                if state.predecessor_error.is_none() {
-                    state.predecessor_error = Some(inherited_error(error));
-                }
+            if result.is_err() && state.predecessor_error.is_none() {
+                state.predecessor_error = Some(PRODUCER_FAILED);
             }
             state.remaining -= 1;
             state.claim()

drivers/gpu/drm/asahi/g17/dependency.rs

@@ -9,6 +9,7 @@ use super::{
 use kernel::prelude::*;
 
 /// A failed producer does not establish a fault in its consumer's VM.
+#[cfg(CONFIG_DRM_ASAHI_G15)]
 pub(crate) fn inherited_error(error: Error) -> Error {
     if error == EIO || error == ENODEV || error == ETIMEDOUT {
         ECANCELED
@@ -17,11 +18,16 @@ pub(crate) fn inherited_error(error: Error) -> Error {
     }
 }
 
+/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its
+/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with
+/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too.
+pub(crate) const PRODUCER_FAILED: Error = ECANCELED;
+
 /// A replay's original host inputs must still be signalled. Treat an unmet
 /// input as a failed replay, never as a wait under the device mutex.
 pub(crate) fn replay_status(status: i32) -> Result {
     if status == 0 { Err(EBUSY) }
-    else if status < 0 { Err(inherited_error(Error::from_errno(status))) }
+    else if status < 0 { Err(PRODUCER_FAILED) }
     else { Ok(()) }
 }
 
@@ -195,4 +201,15 @@ mod tests {
         record.note(5, 1, &Probe::new(true));
         assert_eq!(earliest(&mut record), None);
     }
+
+    #[test]
+    fn failed_predecessor_cannot_be_bypassed() {
+        let mut frontier = Frontier::new();
+        for error in [ECANCELED, EIO, ETIMEDOUT] {
+            assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED));
+        }
+        assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(()));
+        assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(()));
+        assert_eq!(frontier.count, 0);
+    }
 }

drivers/gpu/drm/asahi/g17/fence.rs

@@ -180,7 +180,7 @@ impl Submission {
             self.remaining.load(Ordering::Acquire) == 0 || self.retired.load(Ordering::Acquire);
         let error = self.error.load(Ordering::Acquire);
         if error != 0 {
-            return Err(super::dependency::inherited_error(Error::from_errno(error)));
+            return Err(super::dependency::PRODUCER_FAILED);
         }
         Ok(complete)
     }

drivers/gpu/drm/asahi/g17/job.rs

@@ -177,8 +177,9 @@ impl Packet {
                         kernel::time::msecs_to_jiffies(10) as isize,
                     )
                 };
+                // An interrupted wait is not a producer failure.
                 if waited < 0 {
-                    return Err(super::dependency::inherited_error(Error::from_errno(waited as i32)));
+                    return Err(Error::from_errno(waited as i32));
                 }
                 if waited > 0 { break; }
             }
@@ -257,7 +258,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 {
 fn check_status(fence: &Fence) -> Result {
     let status = fence_status(fence);
     if status < 0 {
-        Err(super::dependency::inherited_error(Error::from_errno(status)))
+        Err(super::dependency::PRODUCER_FAILED)
     } else {
         Ok(())
     }

drivers/gpu/drm/asahi/g17/preparation.rs

@@ -6,6 +6,7 @@
 //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is
 //! stored after its tentative mappings so those mappings are destroyed before the join completes.
 
+use core::cell::Cell;
 use core::sync::atomic::{AtomicU64, Ordering};
 use kernel::{
     new_condvar, new_mutex,
@@ -24,25 +25,37 @@ const EPOCH: u64 = CLOSED << 3;
 /// packet cancellation, logical close, VM status and device loss even when no
 /// capacity/recovery event arrives. Wakes and nested constructors never renew it.
 pub(super) struct Admission<'a> {
-    deadline: u64,
+    deadline: Cell<u64>,
+    started: u64,
     check: &'a dyn Fn() -> Result,
 }
 
 impl<'a> Admission<'a> {
     pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self {
-        Self { deadline, check }
+        Self { deadline: Cell::new(deadline), started: super::now_ns(), check }
+    }
+
+    /// A recovery is pending or this owner's spared work drains: wait as long as the
+    /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing.
+    pub(super) fn extend_for_recovery(&self) {
+        let limit = self
+            .started
+            .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000);
+        if self.deadline.get() < limit {
+            self.deadline.set(limit);
+        }
     }
 
     pub(super) fn check(&self) -> Result {
         (self.check)()?;
-        if super::now_ns() >= self.deadline { return Err(EBUSY); }
+        if super::now_ns() >= self.deadline.get() { return Err(EBUSY); }
         Ok(())
     }
 
     /// Ignore condvar remaining-time reports: only the monotonic deadline counts.
     pub(super) fn slice_ms(&self) -> Result<u32> {
         self.check()?;
-        let remaining = self.deadline.saturating_sub(super::now_ns());
+        let remaining = self.deadline.get().saturating_sub(super::now_ns());
         if remaining == 0 { return Err(EBUSY); }
         Ok(remaining.div_ceil(1_000_000).min(10) as u32)
     }
@@ -137,19 +150,20 @@ impl Gate {
         }
     }
 
-    /// Packet admission may wait for recovery only within its original budget.
+    /// Only a recovery closes the gate: packet admission waits for it up to
+    /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery).
     /// Called with no device mutex or existing preparation lease.
     pub(super) fn enter_until(self: &Arc<Self>, admission: &Admission<'_>) -> Result<Lease> {
         admission.check()?;
         match self.try_enter() {
-            Err(error) if error == EAGAIN => {}
+            Err(error) if error == EAGAIN => admission.extend_for_recovery(),
             result => return result,
         }
         let mut wait = self.wait.lock();
         loop {
             admission.check()?;
             match self.try_enter() {
-                Err(error) if error == EAGAIN => {}
+                Err(error) if error == EAGAIN => admission.extend_for_recovery(),
                 result => return result,
             }
             let _ = self.changed.wait_timeout(
@@ -322,6 +336,20 @@ mod growth_gate_tests {
 mod compute_deadline_tests {
     use super::*;
 
+    #[test]
+    fn a_recovery_extends_the_deadline_once_from_the_start() {
+        let check = || Ok(());
+        let admission = Admission::new(super::super::now_ns(), &check);
+        assert_eq!(admission.check(), Err(EBUSY));
+        admission.extend_for_recovery();
+        assert_eq!(admission.check(), Ok(()));
+        let limit = admission.started
+            + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000;
+        assert_eq!(admission.deadline.get(), limit);
+        admission.extend_for_recovery();
+        assert_eq!(admission.deadline.get(), limit);
+    }
+
     #[test]
     fn spent_budget_refuses_even_an_open_gate() {
         let gate = Gate::new().unwrap();
@@ -332,13 +360,16 @@ mod compute_deadline_tests {
     }
 
     #[test]
-    fn closed_gate_expires_without_any_recovery_wake() {
+    fn closed_gate_waits_past_the_budget_for_recovery() {
         let gate = Gate::new().unwrap();
         gate.close();
         let start = super::super::now_ns();
-        let check = || Ok(());
+        // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting.
+        let check = || {
+            if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) }
+        };
         let admission = Admission::new(start + 20_000_000, &check);
-        assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY));
+        assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED));
         assert!(super::super::now_ns() - start < 500_000_000);
         gate.wait_drained();
         assert!(gate.reopen());

drivers/gpu/drm/asahi/g17/queue/admission.rs

@@ -494,6 +494,16 @@ impl Pool {
             }
         }
     }
+    /// The owner's pair is quarantined behind another VM's recovery (spared) and drains.
+    pub(crate) fn spared_quarantined(&self, owner: u64) -> bool {
+        let state = self.state.lock();
+        state
+            .owners
+            .iter()
+            .position(|id| *id == owner)
+            .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0)
+    }
+
     pub(crate) fn acquire_render(
         self: &Arc<Self>,
         owner: u64,
@@ -525,17 +535,22 @@ impl Pool {
                 .position(|id| *id == owner)
                 .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0)
             {
-                if now >= quarantine_deadline {
-                    // Behind an innocent pair's quarantine (another VM wedged the GPU) only this
-                    // publication fails, not the VM: it recovers once the pair drains.
-                    let spared = state
-                        .owners
-                        .iter()
-                        .position(|id| *id == owner)
-                        .is_some_and(|slot| state.spared & (1u64 << slot) != 0);
-                    return Err(if spared { EBUSY } else { EIO });
+                // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the
+                // pair to drain until the caller's deadline, which the caller extends while it
+                // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second.
+                let spared = state
+                    .owners
+                    .iter()
+                    .position(|id| *id == owner)
+                    .is_some_and(|slot| state.spared & (1u64 << slot) != 0);
+                if now >= quarantine_deadline && !spared {
+                    return Err(EIO);
                 }
-                let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?);
+                let slice = if spared {
+                    remaining_ms
+                } else {
+                    remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?)
+                };
                 if matches!(self.released.wait_interruptible_timeout(
                     &mut state, kernel::time::msecs_to_jiffies(slice)
                 ), CondVarTimeoutResult::Signal { .. }) {
@@ -1001,10 +1016,13 @@ mod fault_capacity_tests {
         let queue = pool.reserve_queue().unwrap();
         let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap();
         let bit = 1u64 << lease.slot();
+        assert!(!pool.spared_quarantined(1));
         lease.quarantine(true);
         assert_ne!(pool.state.lock().spared & bit, 0);
+        assert!(pool.spared_quarantined(1));
         lease.quarantine(false);
         assert_eq!(pool.state.lock().spared & bit, 0);
+        assert!(!pool.spared_quarantined(1));
         // Once guilty, a later spared failure does not make it spared again.
         lease.quarantine(true);
         assert_eq!(pool.state.lock().spared & bit, 0);

drivers/gpu/drm/asahi/g17/queue/compute.rs

@@ -2056,9 +2056,7 @@ impl Queue {
         let ticket = active.ticket.ok_or(EIO)?;
         let state = crate::g17::job::fence_status(&active.packet.completion.fence());
         if state < 0 {
-            return Err(crate::g17::dependency::inherited_error(Error::from_errno(
-                state,
-            )));
+            return Err(crate::g17::dependency::PRODUCER_FAILED);
         }
         if state > 0 {
             return Ok(None);

drivers/gpu/drm/asahi/g17/recovery/safety.rs

@@ -126,5 +126,6 @@ mod tests {
         assert_eq!(replay_status(1), Ok(()));
         assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED));
         assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED));
+        assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED));
     }
 }

drivers/gpu/drm/asahi/g17/runtime.rs

@@ -1756,8 +1756,10 @@ impl Backend {
                 }
             }
             // Other owners continue submitting while this owner waits for its
-            // queue-local drain witness. No new command has been accepted: the
-            // spared queue and its retained work stay intact past the deadline.
+            // queue-local drain witness. No new command has been accepted; if the
+            // admission deadline passes, the spared queue and its retained work stay
+            // intact until a later admission observes its witness.
+            admission.extend_for_recovery();
             self.shared.queue_events();
             self.wait_release(&self.shared.progress, observed, admission)?;
         }
@@ -2090,6 +2092,13 @@ impl Backend {
             admission.check()?;
             let observed = progress.observe();
             if !self.sources_published(packet)? {
+                let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| {
+                    firmware.recovery.pending()
+                        || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0
+                });
+                if recovering {
+                    admission.extend_for_recovery();
+                }
                 self.shared.queue_events();
                 self.wait_release(progress, observed, &admission)?;
                 continue;
@@ -2131,6 +2140,7 @@ impl Backend {
             }
             let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?;
             if firmware.recovery.pending() || recovery_state != 0 {
+                admission.extend_for_recovery();
                 drop(state);
                 drop(lease);
                 self.shared.queue_events();
@@ -2162,6 +2172,11 @@ impl Backend {
             {
                 // Recovery or backpressure: nothing is firmware-visible. Other
                 // queues keep submitting and the event worker retires meanwhile.
+                if firmware.recovery.pending()
+                    || super::recovery::Memory::recovery_state(&firmware.init)? != 0
+                {
+                    admission.extend_for_recovery();
+                }
                 drop(state);
                 drop(lease);
                 self.shared.queue_events();