drm/asahi: admit authorized HIGH queues on Extended T8122

drivers/gpu/drm/asahi/file.rs

@@ -1350,9 +1350,15 @@ impl File {
             cls_pr_debug!(Errors, "queue_create: Invalid priority\n");
             return Err(EINVAL);
         }
-        // HIGH has only been run on G17; G13 and G14 keep refusing it.
+        // Keep support tied to a backend and admitted profile. G13/G14 and
+        // unqualified M3 profiles retain their previous rejection.
+        let high_supported = reg_data.gpu.as_any().is::<crate::g17::Gpu>();
+        #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
+        let high_supported = high_supported || reg_data.gpu.as_any()
+            .downcast_ref::<crate::m3::m3_drm::Backend>()
+            .is_some_and(|gpu| gpu.supports_high_priority());
         if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH
-            && !reg_data.gpu.as_any().is::<crate::g17::Gpu>()
+            && !high_supported
         {
             cls_pr_debug!(
                 Errors,

drivers/gpu/drm/asahi/m3/m3_completion.rs

@@ -24,10 +24,9 @@ impl WorkItem for Completion {
     type Pointer = Arc<Self>;
     fn run(owner: Arc<Self>) {
         use crate::m3::m3_runtime::Service;
-        // Snapshots after a notification before sleeping for the next one: the completion
-        // stamps can trail the firmware's event message.
-        const TRAILING_POLLS: u32 = 4;
-        let (mut trailing, mut waited_at) = (0, u64::MAX);
+        // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep.
+        // Check immediately on IRQ, with backoff for trailing coherent witnesses.
+        let mut policy = super::m3_completion_policy::WaitPolicy::new();
         loop {
             let (outcome, events) = {
                 let mut guard = owner.shared.lock();
@@ -37,23 +36,20 @@ impl WorkItem for Completion {
                 // Preserve the root-only CPU-overlap diagnostic: disabled waits retain
                 // the runtime lock so VM/timestamp preparation cannot overlap the GPU.
                 if let Service::Waiting(messages)=outcome {
-                    if !events.cpu_overlap() {events.wait_past(messages);continue;}
+                    if !events.cpu_overlap() {
+                        events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
+                        continue;
+                    }
                 }
                 (outcome,events)
             };
             match outcome {
                 Service::Idle => break,
-                Service::Progress => { trailing = 0; waited_at = u64::MAX; }
+                Service::Progress => policy.progress(),
                 Service::Waiting(messages) => {
-                    if messages != waited_at { waited_at = messages; trailing = 0; }
-                    if trailing < TRAILING_POLLS {
-                        trailing += 1;
-                        kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5));
-                    } else {
-                        // Until the next firmware event, or a bounded 100 us resnapshot.
-                        events.wait_past(messages);
-                        trailing = 0;
-                    }
+                    // IRQs and timer expiry alone are not retirement progress.
+                    // Keep backing off until service publishes or retires work.
+                    events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
                 }
             }
         }

drivers/gpu/drm/asahi/m3/m3_completion_policy.rs

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Bounded fallback for witnesses which can become visible after the last IRQ.
+//!
+//! Notifications request a fresh snapshot, but do not prove witness progress.
+//! Only publication or retirement resets this policy. In particular, neither a
+//! timer wake nor a stream of early firmware notifications renews a spin budget.
+
+pub(crate) struct WaitPolicy {
+    next_us: u64,
+}
+
+impl WaitPolicy {
+    pub(crate) const fn new() -> Self {
+        Self { next_us: 25 }
+    }
+
+    pub(crate) fn progress(&mut self) {
+        self.next_us = 25;
+    }
+
+    /// No deliberate busy delay: IRQs still wake each sleep immediately.
+    /// The caller supplies the root-selected ceiling (25..=1000 us).
+    pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 {
+        let wait = self.next_us.min(ceiling_us.clamp(25, 1000));
+        self.next_us = (wait * 2).min(1000);
+        wait
+    }
+}
+
+#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))]
+mod tests {
+    use super::WaitPolicy;
+
+    #[test]
+    fn no_progress_backs_off_and_stays_bounded() {
+        let mut policy = WaitPolicy::new();
+        for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] {
+            assert_eq!(policy.next_wait_us(1000), expected);
+        }
+        for _ in 0..100_000 {
+            assert_eq!(policy.next_wait_us(1000), 1000);
+        }
+    }
+
+    #[test]
+    fn publication_or_retirement_restarts_fast_detection() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        policy.progress();
+        assert_eq!(policy.next_wait_us(1000), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+    }
+
+    #[test]
+    fn live_ceiling_changes_apply_to_the_next_wait() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        assert_eq!(policy.next_wait_us(25), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+        assert_eq!(policy.next_wait_us(100), 100);
+        assert_eq!(policy.next_wait_us(100), 100);
+    }
+
+    #[test]
+    fn fallback_keeps_timeout_checks_frequent_without_any_irq() {
+        let mut policy = WaitPolicy::new();
+        let mut elapsed_us = 0;
+        while elapsed_us < 2_000_000 {
+            let wait = policy.next_wait_us(1000);
+            assert!(wait <= 1000);
+            elapsed_us += wait;
+        }
+        // This bounds the requested delay, not scheduler dispatch latency.
+        assert!(elapsed_us < 2_001_000);
+    }
+}

drivers/gpu/drm/asahi/m3/m3_drm.rs

@@ -38,7 +38,7 @@ struct CompletionView(Arc<crate::m3::m3_rtkit::EventWait>);
 impl kernel::debugfs::Writer for CompletionView {
     fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
         let (us, irqs, sleeps, timers, preparations) = self.0.snapshot();
-        writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}",
+        writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}",
                  us, irqs, sleeps, timers, preparations)
     }
 }
@@ -96,6 +96,11 @@ pub(crate) struct Backend {
 }
 
 impl Backend {
+    /// HIGH promises host publication ordering, not firmware preemption.
+    pub(crate) fn supports_high_priority(&self) -> bool {
+        self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended
+    }
+
     /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the
     /// caller registers afterwards.
     pub(crate) fn start(

drivers/gpu/drm/asahi/m3/m3_rtkit.rs

@@ -38,7 +38,7 @@ impl EventWait {
         Arc::pin_init(
             pin_init!(EventWait {
                 count: AtomicU64::new(0),
-                wait_us: AtomicU64::new(100),
+                wait_us: AtomicU64::new(1000),
                 overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()),
                 cpu_preparations: AtomicU64::new(0),
                 sleeps: AtomicU64::new(0),
@@ -55,7 +55,7 @@ impl EventWait {
         self.arrived.complete();
     }
 
-    /// Diagnostic tuning changes only the bounded resnapshot cadence, never
+    /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never
     /// stamps, pipe retirement, event counts, fences or the two-second timeout.
     /// Atomic publication permits root to compare policies in the same boot.
     pub(crate) fn set_wait_us(&self, us: u64) -> Result {
@@ -64,6 +64,10 @@ impl EventWait {
         Ok(())
     }
 
+    pub(crate) fn wait_ceiling_us(&self) -> u64 {
+        self.wait_us.load(Ordering::Relaxed)
+    }
+
     pub(crate) fn cpu_overlap(&self) -> bool {
         self.overlap_cpu.load(Ordering::Relaxed)
     }
@@ -82,11 +86,11 @@ impl EventWait {
     }
 
     /// Sleep until a notification later than `seen` has been recorded or
-    /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250,
+    /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250,
     /// even when the GPU stamp arrives just after its notification. The timer
     /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting.
     /// Only the single completion worker may call this method.
-    pub(crate) fn wait_past(self: &Arc<Self>, seen: u64) -> bool {
+    pub(crate) fn wait_past(self: &Arc<Self>, seen: u64, wait_us: u64) -> bool {
         // Events can arrive while the worker is polling. Discard stale wakeup
         // tokens before checking the sequence. A concurrent event either moves
         // the sequence or leaves a token, including between the check and wait.
@@ -98,7 +102,7 @@ impl EventWait {
                 // on every return, before a subsequent wait can arm the timer.
                 self.sleeps.fetch_add(1, Ordering::Relaxed);
                 let timer = self.clone().start(Delta::from_micros(
-                    self.wait_us.load(Ordering::Relaxed) as i64));
+                    wait_us.clamp(25, 1000) as i64));
                 // Retain one tick as a backup bound; normally either the real
                 // interrupt or our high-resolution timer supplies the token.
                 self.arrived.wait_for_completion_timeout(1);

drivers/gpu/drm/asahi/m3/m3_runtime.rs

@@ -1079,6 +1079,8 @@ impl Runtime {
                 crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e);
                 state.health.mark_failed();
                 if device.stop_asc().is_err() {
+                    // Without module unloading, the C helper is an unbound no-op.
+                    #[cfg(CONFIG_MODULE_UNLOAD)]
                     unsafe {kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr())};
                     core::mem::forget(transport);core::mem::forget(state);
                     core::mem::forget(adopted);
@@ -1348,6 +1350,7 @@ impl Drop for Runtime {
         for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));}
         if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) {
             // SAFETY: a live module reference pins callbacks with retained DMA.
+            #[cfg(CONFIG_MODULE_UNLOAD)]
             unsafe { kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr()) };
             return;
         }

drivers/gpu/drm/asahi/m3/mod.rs

@@ -133,6 +133,8 @@ mod t8122_knobs;
 #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
 mod t8122_start;
 
+mod m3_completion_policy;
+
 mod m3_compute_layout;
 
 mod m3_compute_sequence;

tools/asahi/m3-fence-latency.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Join existing M3 timing logs. Input must be one boot, captured without gaps."""
+import collections
+import json
+import math
+import re
+import sys
+
+milestones = {}
+packets = collections.defaultdict(list)
+fences = []
+invalid = collections.Counter()
+for line in open(sys.argv[1]):
+    fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)}
+    if 'M3_BATCH_MILESTONE ' in line:
+        key = (fields['vm'], fields['publication_tick'])
+        if key in milestones:
+            invalid['duplicate_milestone'] += 1
+        milestones[key] = fields
+    elif 'M3_BATCH_PACKET ' in line:
+        packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command']))
+    elif 'M3_PACKET_FENCE ' in line:
+        fences.append(fields)
+values = []
+previous_signal = {}
+for f in sorted(fences, key=lambda f: f['signal_before_tick']):
+    identity = (f['vm'], f['packet'])
+    previous = previous_signal.get(identity, 0)
+    previous_signal[identity] = f['signal_after_tick']
+    if f['status'] != 0:
+        invalid['error_fence'] += 1
+        continue
+    linked = packets.get((f['vm'], f['packet']), [])
+    # Pointer reuse is disambiguated by the completed fence's physical counter.
+    candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1]
+    if not candidates:
+        invalid['missing_final_command'] += 1
+        continue
+    final_tick, _ = max(candidates)
+    batch = milestones.get((f['vm'], final_tick))
+    if not batch or batch['mixed'] or batch['render_end_tick'] <= 0:
+        invalid['missing_or_non_render_end'] += 1
+        continue
+    end = batch['render_end_tick']
+    before, after = f['signal_before_tick'], f['signal_after_tick']
+    if not (final_tick <= end <= before <= after):
+        invalid['invalid_counter_order'] += 1
+        continue
+    # GPU execution end is earlier than full coherent retirement; this includes
+    # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done.
+    values.append(((before - end) / 24, (after - end) / 24))
+
+def percentiles(data):
+    ordered = sorted(data)
+    return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {}
+
+print(json.dumps({'render_fences': len(values), 'invalid': invalid,
+                  'gpu_end_to_signal_before_us': percentiles([a for a, b in values]),
+                  'gpu_end_to_signal_after_us': percentiles([b for a, b in values]),
+                  'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2))
+if not values:
+    sys.exit('No valid render fences: do not report latency from this capture')

tools/asahi/m3-runtime-tests.rs

@@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() {
         }
     }
 }
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"]
+mod m3_completion_policy;

tools/asahi/queue-priority-tests.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Compile the production queue admission gate with controlled credentials/backends."""
+from pathlib import Path
+import subprocess
+import tempfile
+root = Path(__file__).resolve().parents[2]
+s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text()
+a = s.index('        if data.flags != 0', s.index('pub(crate) fn queue_create('))
+b = s.index('        let queues_xa', a)
+gate = s[a:b]
+predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n    }', 1)[0]
+submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text()
+ordering = submit.split('        let urgency=', 1)[1].split('        Ok(Self', 1)[0]
+ordering = 'let urgency=' + ordering
+assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s
+unit = r'''
+#![allow(dead_code, non_upper_case_globals)]
+use std::any::Any;
+const EINVAL:i32=22; const EACCES:i32=13;
+macro_rules! cls_pr_debug { ($($arg:tt)*) => {} }
+mod uapi {
+ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3;
+ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2;
+}
+mod bindings { pub const CAP_SYS_NICE:u32=23;
+ pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) }
+ thread_local! {pub static CAP:std::cell::Cell<bool>=const {std::cell::Cell::new(false)};}
+}
+mod g17 {pub struct Gpu;}
+mod m3 {pub mod m3_drm {
+ #[derive(PartialEq)] pub enum Profile {Legacy, Extended}
+ pub struct Soc {pub chip_id:u32}
+ pub struct Backend {pub soc:Soc,pub profile:Profile}
+ impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }}
+}}
+struct Gpu(Box<dyn Any>); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}}
+struct Reg {gpu:Gpu}
+struct Data {flags:u32,priority:u32}
+struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}}
+fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) }
+mod sched {
+ #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel}
+}
+fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> {
+ ORDERING
+ Ok((urgency,priority))
+}
+fn main() {
+ use m3::m3_drm::*;
+ let mut n=0;
+ for backend in 0..5 { for cap in [false,true] { for master in [false,true] {
+  let gpu:Box<dyn Any>=match backend {
+   0=>Box::new(g17::Gpu),
+   1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}),
+   2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}),
+   3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}),
+   _=>Box::new(())};
+  let reg=Reg{gpu:Gpu(gpu)};
+  bindings::CAP.with(|c|c.set(cap));
+  for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] {
+   let expected=if flags!=0 || priority>=3 {Err(EINVAL)}
+    else if priority!=2 {Ok(())}
+    else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)}
+    else if cap||master {Ok(())} else {Err(EACCES)};
+   assert_eq!(admit(&reg,&Data{flags,priority},&File(master)),expected,
+    "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1;
+  }}
+ }}}
+ for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] {
+  assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected)));
+ }
+ assert_eq!(ordering(4),Err(EINVAL));
+ println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed");
+}
+'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering)
+with tempfile.TemporaryDirectory(prefix='queue-priority-') as d:
+    p = Path(d) / 'gate.rs'
+    p.write_text(unit)
+    for enabled in [False, True]:
+        subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True)
+        # Without M3 compiled only the G17 backend is admitted.
+        subprocess.run([str(p.with_suffix(''))], check=True)
+print('M3 enabled/disabled gate compilation passed')