diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..a09fa33ed02a4 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1350,9 +1350,15 @@ impl File { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. + // Keep support tied to a backend and admitted profile. G13/G14 and + // unqualified M3 profiles retain their previous rejection. + let high_supported = reg_data.gpu.as_any().is::(); + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let high_supported = high_supported || reg_data.gpu.as_any() + .downcast_ref::() + .is_some_and(|gpu| gpu.supports_high_priority()); if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() + && !high_supported { cls_pr_debug!( Errors, diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047e..387aaeca1b6c6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 0000000000000..9c0da8c3246e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 1117424aa1662..b11b612184ed9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } @@ -96,6 +96,11 @@ pub(crate) struct Backend { } impl Backend { + /// HIGH promises host publication ordering, not firmware preemption. + pub(crate) fn supports_high_priority(&self) -> bool { + self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended + } + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the /// caller registers afterwards. pub(crate) fn start( diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d92..ba8ad365ce20a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 082449e34cf40..e34236fd57a3f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1079,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(kernel::module::this_module::().as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1348,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(kernel::module::this_module::().as_ptr()) }; return; } diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647c..eafd5732bceac 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 0000000000000..c8428d65dbf0d --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 91cbefd298fe2..0d6d4e5841194 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { } } } + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy; diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py new file mode 100644 index 0000000000000..1c5b4695c85f7 --- /dev/null +++ b/tools/asahi/queue-priority-tests.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production queue admission gate with controlled credentials/backends.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[2] +s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() +a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) +b = s.index(' let queues_xa', a) +gate = s[a:b] +predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() +ordering = submit.split(' let urgency=', 1)[1].split(' Ok(Self', 1)[0] +ordering = 'let urgency=' + ordering +assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s +unit = r''' +#![allow(dead_code, non_upper_case_globals)] +use std::any::Any; +const EINVAL:i32=22; const EACCES:i32=13; +macro_rules! cls_pr_debug { ($($arg:tt)*) => {} } +mod uapi { + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3; + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; +} +mod bindings { pub const CAP_SYS_NICE:u32=23; + pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) } + thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} +} +mod g17 {pub struct Gpu;} +mod m3 {pub mod m3_drm { + #[derive(PartialEq)] pub enum Profile {Legacy, Extended} + pub struct Soc {pub chip_id:u32} + pub struct Backend {pub soc:Soc,pub profile:Profile} + impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }} +}} +struct Gpu(Box); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}} +struct Reg {gpu:Gpu} +struct Data {flags:u32,priority:u32} +struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} +fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +mod sched { + #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel} +} +fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> { + ORDERING + Ok((urgency,priority)) +} +fn main() { + use m3::m3_drm::*; + let mut n=0; + for backend in 0..5 { for cap in [false,true] { for master in [false,true] { + let gpu:Box=match backend { + 0=>Box::new(g17::Gpu), + 1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}), + 2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}), + 3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}), + _=>Box::new(())}; + let reg=Reg{gpu:Gpu(gpu)}; + bindings::CAP.with(|c|c.set(cap)); + for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] { + let expected=if flags!=0 || priority>=3 {Err(EINVAL)} + else if priority!=2 {Ok(())} + else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)} + else if cap||master {Ok(())} else {Err(EACCES)}; + assert_eq!(admit(®,&Data{flags,priority},&File(master)),expected, + "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; + }} + }}} + for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] { + assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected))); + } + assert_eq!(ordering(4),Err(EINVAL)); + println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed"); +} +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering) +with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: + p = Path(d) / 'gate.rs' + p.write_text(unit) + for enabled in [False, True]: + subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True) + # Without M3 compiled only the G17 backend is admitted. + subprocess.run([str(p.with_suffix(''))], check=True) +print('M3 enabled/disabled gate compilation passed') diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index ca9bae81672a7..470058e6a369b 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'queue-priority-tests.py': [([], PASS)], # The old revision reports a DART fault that clears before the second latch read as success. 'ane-tm-transient-fault-tests.py': [([], PASS), (['--old-ref', 'b50066848322c6799c1433ee3a358aab2e9faabb'], PASS)],