From 7ccdfe85843df16b40bbb5f0080b2d288ae0b8bc Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sun, 25 Oct 2026 23:12:44 +0100 Subject: [PATCH 1/6] drm/asahi: wrap G15 stamps through zero Include zero in the firmware stamp cycle. Keep absent dependencies and completed progress separate from wrapped zero stamps. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit d82144ad56a8ac7e75d74294e051683efccbb2a0) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/g15/encode.rs | 5 +-- drivers/gpu/drm/asahi/g15/engine.rs | 16 +++----- drivers/gpu/drm/asahi/g15/poll.rs | 33 ++++++++------- drivers/gpu/drm/asahi/g15/recovery/logic.rs | 41 +++++++++++++++---- .../selftests/drivers/gpu/asahi-g15/run.py | 17 ++++++++ 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index e46ecd05bdf78b..64aebf32548914 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -64,10 +64,9 @@ const fn align(n: u64, a: u64) -> u64 { } /// Firmware stamp value of ordinal `n` on the queue with this seed. -/// Stamps step by 0x100 through 0xffffff values and never read 0. +/// Stamps step by 0x100, including zero when the counter wraps. pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { - let base = (seed / 0x100) as u64; - ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 + seed.wrapping_add((n as u32).wrapping_mul(0x100)) } /// Compute stamp of ordinal `n`. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 33974222afe9b0..74d9590baf0c48 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -155,10 +155,10 @@ pub(crate) struct Flight { pub(crate) fs_start: u32, /// Channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// Compute member j's FW stamp target, and the FS target it waits on directly: /// its predecessors in the prefix (for the compute-hang check). pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], - pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [Option; sizes::COMPUTE_SLOTS], /// Compute slots, pass slots and clustering buffers it holds. pub(crate) claims: Claims, /// The prefix rebuilds the render allocators (priming); nothing is published behind it. @@ -189,7 +189,7 @@ impl Flight { fs_start: 0, rec: [0; 3], cstamp: [0; sizes::COMPUTE_SLOTS], - cwait: [0; sizes::COMPUTE_SLOTS], + cwait: [None; sizes::COMPUTE_SLOTS], claims: Claims::new(), rebuild: false, doomed: false, @@ -387,7 +387,7 @@ pub(crate) struct Engine { pub(crate) chan_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, - pub(crate) progress: [u32; 3], + pub(crate) progress: [Option; 3], pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, @@ -425,7 +425,7 @@ impl Engine { rebuild_live: false, chan_wait: false, progress_flight: 0, - progress: [0; 3], + progress: [None; 3], stats: Stats::default(), prof: Prof::default(), debugfs: None, @@ -1191,11 +1191,7 @@ impl Firmware { } let j = pf.computes as usize; f.cstamp[j] = t; - f.cwait[j] = if pf.prev == Prev::Render { - pf.last_fs - } else { - 0 - }; + f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs); g.ring.w64(8 * pf.c_head as usize, s.work.va)?; pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; lap.lap(&mut self.engine.prof.write_ns); diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 751bccd07b119e..9246384e677732 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -189,10 +189,7 @@ impl Shared { } } -/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. -pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) -} +pub(crate) use super::recovery::logic::stamp_reached; /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { @@ -498,23 +495,25 @@ impl Firmware { return; }; let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); - let (mut c, mut ta, mut fs) = (0, 0, 0); + let pending = super::recovery::logic::pending_stamp; + let (mut c, mut ta, mut fs) = (None, None, None); if f.computes != 0 { - c = g.fw_stamp.r32(0).unwrap_or(0); - if stamp_reached(c, f.target, e.target) { - c = 0; // its computes are done: later work does not count - } + c = g + .fw_stamp + .r32(0) + .ok() + .and_then(|v| pending(v, f.target, e.target)); } if f.renders != 0 { - ta = r.fw_stamp[0].r32(0).unwrap_or(0); - fs = r.fw_stamp[1].r32(0).unwrap_or(0); - if stamp_reached(ta, f.ta_target, e.ta_target) { - ta = 0; - } - if stamp_reached(fs, f.fs_target, e.fs_target) { - fs = 0; - } + ta = r.fw_stamp[0] + .r32(0) + .ok() + .and_then(|v| pending(v, f.ta_target, e.ta_target)); + fs = r.fw_stamp[1] + .r32(0) + .ok() + .and_then(|v| pending(v, f.fs_target, e.fs_target)); } let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; let published = f.published_ns; diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs index 6d36d1d6550d5a..767fb7c9529372 100644 --- a/drivers/gpu/drm/asahi/g15/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -37,9 +37,18 @@ pub(crate) const DRAIN_POLLS: u32 = 1000; pub(crate) const FURTHER_HALTS: u32 = 4; /// Stamp cell `obs` reached the own target `own`, bounded by the newest published target -/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +/// `tail`. Written stamps have a zero low byte, including at counter wrap. pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) + obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// The position of unfinished work; zero is a valid position after stamp wrap. +pub(crate) const fn pending_stamp(obs: u32, own: u32, tail: u32) -> Option { + if stamp_reached(obs, own, tail) { + None + } else { + Some(obs) + } } /// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within @@ -73,9 +82,9 @@ pub(crate) struct Receipts { pub(crate) fs_start: u32, /// Host channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + /// Compute member j's FW stamp target, and the FS target it waits on directly. pub(crate) cstamp: [u32; 16], - pub(crate) cwait: [u32; 16], + pub(crate) cwait: [Option; 16], } /// The newest published receipts. @@ -136,8 +145,10 @@ pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { if j == computes { return false; } - if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { - return false; // still waiting on a render of the prefix + if let Some(target) = f.cwait[j] { + if !stamp_reached(o.fs_stamp, target, t.fs_target) { + return false; // still waiting on a render of the prefix + } } if o.consumer[COMPUTE] != f.rec[COMPUTE] { return false; // its record not consumed yet @@ -332,10 +343,20 @@ mod tests { assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); // Across the u32 wrap. let own = 0xffffff00u32; + assert!(stamp_reached(0, own, 0x00000200)); + assert!(stamp_reached(0, 0, 0)); + assert!(!stamp_reached(0xffffff00, 0, 0x100)); assert!(stamp_reached(0x00000100, own, 0x00000200)); assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); } + #[test] + fn progress_at_zero_wrap() { + assert_eq!(pending_stamp(0, 0x100, 0x200), Some(0)); + assert_eq!(pending_stamp(0, 0, 0x100), None); + assert_eq!(pending_stamp(0xffffff00, 0, 0x100), Some(0xffffff00)); + } + #[test] fn rings() { // Entries 10..14 of a 0x500 ring, tail at 20. @@ -401,13 +422,19 @@ mod tests { assert!(!compute_hang(&f, &t, &o)); o.compute_read = 15; // The running compute waits on an FS of the prefix that has not completed. - f.cwait[1] = 0x3d000300; + f.cwait[1] = Some(0x3d000300); o.fs_stamp = 0x3d000200; let mut t2 = t; t2.fs_target = 0x3d000400; assert!(!compute_hang(&f, &t2, &o)); o.fs_stamp = 0x3d000300; assert!(compute_hang(&f, &t2, &o)); + f.cwait[1] = Some(0); + t2.fs_target = 0x100; + o.fs_stamp = 0xffffff00; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0; + assert!(compute_hang(&f, &t2, &o)); // No compute in the prefix. f.computes = 0; assert!(!compute_hang(&f, &t, &o)); diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index be10389b611ed5..cc636fae0c39c8 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -303,6 +303,23 @@ def main(): # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. harness = RUST_PREFIX + f'\n#[path = "{G15 / "gate.rs"}"] mod gate;\n' + encode = (G15 / "encode.rs").read_text() + start = encode.index("pub(crate) const fn stamp_seed(") + end = encode.index("\n}", start) + 2 + harness += encode[start:end] + ''' +#[test] +fn firmware_stamp_wrap() { + for seed in [0xc1000000u32, 0x7a000000, 0x3d000000] { + let wrap = ((1u64 << 32) - seed as u64) / 0x100; + assert_eq!(stamp_seed(seed, 0), seed); + assert_eq!(stamp_seed(seed, wrap - 1), 0xffffff00); + assert_eq!(stamp_seed(seed, wrap), 0); + assert_eq!(stamp_seed(seed, wrap + 1), 0x100); + assert_eq!(stamp_seed(seed, 1 << 24), seed); + assert_eq!(stamp_seed(seed, u64::MAX), seed.wrapping_sub(0x100)); + } +} +''' harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' From bb3ce596fa1e53595c929cab8e7748e2952ec111 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sun, 25 Oct 2026 23:12:44 +0100 Subject: [PATCH 2/6] drm/asahi: bind G15 slots at publication Idle VMs must not reserve hardware ASIDs. Bind without waiting at publication, release at successful retirement, and retain failed published jobs. Logical VM creation is limited by memory rather than slot count. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 8525dd8e380a708356ec1ffaef0bbdf266cb328b) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/g15/engine.rs | 42 ++++++++++--- drivers/gpu/drm/asahi/g15/hw.rs | 2 - drivers/gpu/drm/asahi/g15/job.rs | 47 +++++++++++--- drivers/gpu/drm/asahi/g15/mod.rs | 2 +- drivers/gpu/drm/asahi/g15/poll.rs | 16 +++-- drivers/gpu/drm/asahi/g15/pools.rs | 6 +- drivers/gpu/drm/asahi/g15/recovery.rs | 6 +- drivers/gpu/drm/asahi/g15/uat.rs | 5 +- drivers/gpu/drm/asahi/g15/vm.rs | 89 +++++---------------------- drivers/gpu/drm/asahi/mmu.rs | 15 ++++- drivers/gpu/drm/asahi/slotalloc.rs | 17 +++++ 11 files changed, 143 insertions(+), 104 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 74d9590baf0c48..ba6408d230da54 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -385,6 +385,8 @@ pub(crate) struct Engine { pub(crate) rebuild_live: bool, /// A ready prefix waits only for a record's consumption: repoll sooner. pub(crate) chan_wait: bool, + /// A ready job waits for an ASID held by a completing failed job. + pub(crate) slot_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, pub(crate) progress: [Option; 3], @@ -424,6 +426,7 @@ impl Engine { render_registered: false, rebuild_live: false, chan_wait: false, + slot_wait: false, progress_flight: 0, progress: [None; 3], stats: Stats::default(), @@ -546,6 +549,7 @@ impl Firmware { let mut result = Ok(()); self.engine.chan_wait = false; + self.engine.slot_wait = false; // Nothing is published while a hang request is armed. while self.recovery.failed().is_none() && !self.engine.frozen @@ -592,7 +596,7 @@ impl Firmware { dev_err!( self.graph.dev.as_ref(), "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", - job.vm.slot, + job.slot(), e ); complete(done, &job, Err(EIO)); @@ -675,6 +679,15 @@ impl Firmware { /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE /// advanced; any other error means nothing was published. fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + if let Err(error) = job.bind() { + if error == ENOSPC { + // A failed job can keep its binding until scheduler references drop after + // this pass. Leave healthy work pending and retry without the device lock. + self.engine.slot_wait = true; + return Err(EAGAIN); + } + return Err(error); + } let mut f = Flight::new(job.clone()); let result = self.publish_prefix_into(job, &mut f); match result { @@ -682,7 +695,11 @@ impl Firmware { // Never published: its storage is free again, and horizon jobs taken into it // fail with it. self.slots.release(&mut f.claims); + if job.progress.next() == job.progress.sent() { + job.release_binding(); + } for j2 in f.group.iter() { + j2.release_binding(); complete(done, j2, Err(e)); } } @@ -899,11 +916,20 @@ impl Firmware { hz_stop = Some(1); break; } - for k in 0..n2 { - match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, - Command::Render(_) => return Err(EINVAL), + j2.bind()?; + let encoded = (|| { + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, + Command::Render(_) => return Err(EINVAL), + } } + Ok(()) + })(); + if let Err(error) = encoded { + // Not in the horizon yet and nothing of it reached the firmware. + j2.release_binding(); + return Err(error); } computes += n2; j2.progress.set_sent(n2); @@ -1057,7 +1083,7 @@ impl Firmware { Queue, self.graph.dev, "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", - job.vm.slot, + job.slot(), computes, pf.n, pf.c_head, @@ -1095,7 +1121,7 @@ impl Firmware { ) -> Result { let n = pf.n + 1; let t = encode::stamp(n); - let ctx = job.vm.slot; + let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); // Nothing published yet on failure. @@ -1236,7 +1262,7 @@ impl Firmware { let k = f.claims.pass[pf.renders as usize] as usize; let n = pf.draw + 1; let usc = job.queue.usc_base; - let ctx = job.vm.slot; + let ctx = job.slot(); let tta = encode::stamp_seed(ids::TA_SEED, n); let tfs = encode::stamp_seed(ids::FS_SEED, n); // Early tiling: a later render member after a render of the same prefix whose render diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index fd8771d9d3e65a..f231e573c40176 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -514,8 +514,6 @@ pub(crate) mod sizes { pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). pub(crate) const MAX_QUEUES: u32 = 58; - /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). - pub(crate) const MAX_VMS: u32 = 63; } /// Firmware channels, doorbells, seeds and identifiers. diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index a360b58987c815..dbca91ee3202b3 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -22,29 +22,35 @@ //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage //! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its -//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use kernel::{ bindings, c_str, device, dma_fence::{Fence, RawDmaFence}, drm::sched, + new_mutex, prelude::*, - sync::Arc, + sync::{Arc, Mutex}, }; use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; use crate::mmu; /// A job as the engine sees it. +#[pin_data] pub(crate) struct Job { /// Submission ID (debugging). pub(crate) id: u64, /// The submitting queue (priority, service turn, horizon history). pub(crate) queue: Arc, - /// The job's VM (TTBAT slot, status). + /// The job's VM and status. pub(crate) vm: Arc, + /// Hardware context, assigned immediately before the engine publishes the job. + context: AtomicU32, + #[pin] + binding: Mutex>, /// Validated commands, in submission order (at most 64). pub(crate) commands: KVec, /// The job's hardware fence. @@ -73,11 +79,13 @@ impl Job { guard: mmu::VmJobGuard, ) -> Result> { let progress = super::engine::Progress::new(commands.len()); - Ok(Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { id, queue, vm, + context: AtomicU32::new(0), + binding <- new_mutex!(None, "g15::Job::binding"), commands, fence, progress, @@ -85,9 +93,29 @@ impl Job { _guard: guard, accepted: AtomicBool::new(false), started: AtomicBool::new(false), - }, + }), GFP_KERNEL, - )?) + ) + } + + pub(crate) fn slot(&self) -> u32 { + self.context.load(Ordering::Relaxed) + } + + /// Called under the device lock immediately before publication. + pub(crate) fn bind(&self) -> Result { + let mut slot = self.binding.lock(); + if slot.is_none() { + let binding = self.vm.try_bind()?; + self.context.store(binding.slot(), Ordering::Relaxed); + *slot = Some(binding); + } + Ok(()) + } + + /// All commands retired successfully, or nothing of the job was published. + pub(crate) fn release_binding(&self) { + drop(self.binding.lock().take()); } /// Completes the job: latches the VM status on failure, then signals the fence (after every @@ -120,6 +148,9 @@ impl Job { self.vm.status.record(error); } } + if result.is_ok() || !engine { + self.release_binding(); + } self.fence.signal(result); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index af77a54f9c189a..03eefce9a193cc 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -375,7 +375,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 9246384e677732..647c503ba55037 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -117,7 +117,7 @@ impl Shared { if let Err(error) = firmware.publish(&mut done) { self.fail_stop(error); } - if !firmware.engine.flights.is_empty() { + if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait { self.rearm(firmware.engine.chan_wait); } } @@ -215,7 +215,7 @@ impl Firmware { // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. self.note_idle(); self.reclaim_check(); - if self.recovery_idle() { + if self.recovery_idle() && self.engine.pending.is_empty() { return Ok(false); } } @@ -240,7 +240,10 @@ impl Firmware { // at the start of a pass that finds the flights empty. self.note_idle(); published?; - Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + Ok(!self.engine.flights.is_empty() + || self.engine.slot_wait + || self.recovery.needs_repoll() + || self.engine.frozen) } /// Drains the rings and retires finished flights, with up to four trailing 5 us polls @@ -459,6 +462,10 @@ impl Firmware { p.count.store(p.sent(), Ordering::Relaxed); } } + if result.is_ok() && p.next() == p.sent() { + // All published prefixes retired; another VM may run before this job resumes. + job.release_binding(); + } if p.next() < p.sent() || p.sent() < p.count() { return; // more of it published or still to publish } @@ -476,6 +483,7 @@ impl Firmware { match result { Err(error) => complete(done, j2, Err(error)), Ok(()) => { + j2.release_binding(); s.jobs += 1; s.job_cmds += p2.count() as u64; s.job_max = s.job_max.max(p2.count() as u64); @@ -606,7 +614,7 @@ impl Firmware { self.graph.dev.as_ref(), "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", what, - f.map_or(0, |f| f.job.vm.slot), + f.map_or(0, |f| f.job.slot()), e.flights.len(), e.ordinal, e.target, diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 3364b136b9287f..8a3989f723abb7 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -624,8 +624,8 @@ impl Firmware { if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", - vm.slot, + "PB backing: VM {} maps {} chunks with {} flights of another VM\n", + vm.id, self.pools.chunks.len(), others ); @@ -913,7 +913,7 @@ impl Firmware { .flights .iter() .enumerate() - .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.slot() == ctx); // At the maximum, after the refusals, the request is left unanswered: the poll pass // returns, and the host deadline and its recovery path apply to a stalled TA. if self.pools.ledger.exhausted() { diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index 0827385e92c5fa..fc9bba6cbdce3b 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -224,7 +224,7 @@ impl Firmware { for flight in self.engine.flights.iter_mut().skip(1) { flight.doomed = true; } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); let kind = if offset == logic::FRAGMENT_KILL { "fragment" } else { @@ -281,7 +281,7 @@ impl Firmware { if decision.compute_hang { self.report("compute hang"); } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); if decision.class0_hang { dev_warn!( self.dev(), @@ -323,7 +323,7 @@ impl Firmware { /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { let flight = !self.engine.flights.is_empty(); - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); self.report(if flight { "prefix FAILED" } else { diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 7ab3a1919eaecc..b37a651689025c 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,9 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement; failed published jobs retain their slots. +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index e2b9d884bc25b8..14d98bc4d50b1b 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -13,16 +13,11 @@ //! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are //! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed //! ([`VmState::map_heap`], [`VmState::map_chunk`]); -//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] -//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before -//! binding so the slot allocator never blocks). +//! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds +//! its binding through completion; failed published jobs retain theirs until reboot. //! -//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), -//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding -//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the -//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when -//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their -//! VM's slot with them). +//! The registry keeps a VM's state while its client handle is open. Queues and jobs retain +//! closed VMs until their last reference drops. //! //! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by //! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). @@ -30,7 +25,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, Ordering}; +use core::sync::atomic::Ordering; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -39,7 +34,7 @@ use kernel::{ }; use super::{hw, object, Gpu}; -use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu}; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -123,20 +118,22 @@ struct HeapMappings { pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). pub(crate) id: u64, - /// The VM's TTBAT slot (ASID), for the work images' context. - pub(crate) slot: u32, /// The sticky status (shared with `file.rs`). pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, - /// The exclusive slot binding, released before the live count drops. - bind: Option, + /// The adopted UAT, used to bind published jobs without waiting for a slot. + uat: Arc, registry: Arc, /// The PB heap chunks mapped in this VM. heap: Arc, } impl VmState { + pub(crate) fn try_bind(&self) -> Result { + self.uat.try_bind(&self.vm) + } + /// Drops the registry's references to closed VMs (the queues call this when destroyed). pub(crate) fn prune_registry(&self) { self.registry.prune(); @@ -199,39 +196,11 @@ impl VmState { } } -impl Drop for VmState { - fn drop(&mut self) { - mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); - drop(self.bind.take()); - self.registry.live.fetch_sub(1, Ordering::AcqRel); - } -} - -/// Live-VM accounting and the VMs whose client handle is open. +/// The VMs whose client handle is open. #[pin_data] pub(crate) struct Registry { - /// VMs whose client handle may still be open, oldest first. #[pin] vms: Mutex>>, - /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. - live: AtomicU32, -} - -/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. -struct Reservation<'a>(Option<&'a Registry>); - -impl Reservation<'_> { - fn into_state(mut self) { - self.0 = None; - } -} - -impl Drop for Reservation<'_> { - fn drop(&mut self) { - if let Some(registry) = self.0 { - registry.live.fetch_sub(1, Ordering::AcqRel); - } - } } impl Registry { @@ -239,32 +208,14 @@ impl Registry { Arc::pin_init( pin_init!(Self { vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), - live: AtomicU32::new(0), }), GFP_KERNEL, ) } - /// Reserves a place for a new VM after dropping the registry's references to closed VMs. - fn reserve(&self) -> Result> { - self.prune(); - // The shared slot allocator hands out a single slot with robust isolation. - let limit = if *module_parameters::robust_isolation.value() != 0 { - 1 - } else { - hw::sizes::MAX_VMS - }; - self.live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { - (live < limit).then_some(live + 1) - }) - .map_err(|_| ENOSPC)?; - Ok(Reservation(Some(self))) - } - /// Drops the registry's references to VMs whose client handle is closed. Called on VM close /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page - /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + /// tables and bound objects go as soon as its last queue or job is gone. fn prune(&self) { let mut closed = KVec::new(); { @@ -367,7 +318,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { ClientAlias::collect(&firmware.objects)?, ) }; - let reservation = registry.reserve()?; + registry.prune(); let id = gpu.ids.vm.next(); let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; let status = vm.status().ok_or(EINVAL)?.clone(); @@ -380,25 +331,19 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { }), GFP_KERNEL, )?; - // The context names the root only now: everything above is already visible to the walker. - let bind = gpu.uat.bind(&vm)?; - let slot = bind.slot(); - // From here on the state (even one that fails to allocate) gives the place back. - reservation.into_state(); let state = Arc::new( VmState { id, - slot, status, vm: vm.clone(), - bind: Some(bind), + uat: gpu.uat.clone(), registry: registry.clone(), heap, }, GFP_KERNEL, )?; registry.vms.lock().push(state, GFP_KERNEL)?; - mod_pr_debug!("G15: VM {} context {}\n", id, slot); + mod_pr_debug!("G15: VM {} created\n", id); Ok(vm) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bf4a6823024de..c9adb8ba8f3910 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2754,6 +2754,15 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { let mut binding = vm.binding.lock(); if binding.binding.is_none() { @@ -2767,7 +2776,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index fde7470fe57791..a504e7491bfda6 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -216,6 +216,20 @@ impl SlotAllocator { &self, token: Option, cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + self.get_inner_impl(token, cb, true) + } + + /// Gets a slot without waiting; returns ENOSPC when every slot is held. + pub(crate) fn try_get(&self, token: Option) -> Result> { + self.get_inner_impl(token, |_a, _b| Ok(()), false) + } + + fn get_inner_impl( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + wait: bool, ) -> Result> { let mut inner = self.0.inner.lock(); @@ -256,6 +270,9 @@ impl SlotAllocator { } if oldest_time == u64::MAX { + if !wait { + return Err(ENOSPC); + } if first && inner.slot_limit == usize::MAX { pr_warn!( "{}: out of slots, blocking\n", From 289bcca8fd867448c3377072b00f4ec7b7ed441f Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Mon, 26 Oct 2026 03:21:02 +0100 Subject: [PATCH 3/6] drm/asahi: lock G15 mappings once per submit Each range check and stream read relocked the VM and all external BOs. Share one exec lock across validation, preserving permission checks and cache maintenance, and propagate acquisition errors. In profiled 10-second vkQuake runs, kernel CPU time falls from 3.30 to 1.01 seconds. The bounded validation walk holds VM and external BO reservations together, so other users of shared buffers can wait. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 610cc54ad4987dd4bf8948d44e1161b1a260b0e4) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/g15/command.rs | 39 ++++++++++++++----- drivers/gpu/drm/asahi/g15/gate.rs | 12 +++--- drivers/gpu/drm/asahi/mmu.rs | 25 +++++++++++- .../selftests/drivers/gpu/asahi-g15/run.py | 29 +++++++------- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026f..78b5058c8d33f7 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -376,7 +381,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -406,7 +417,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -466,8 +481,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -551,11 +566,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -649,6 +664,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -657,12 +678,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -677,7 +698,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index a6fb20bf156b5c..590f5e6f0a5c11 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -71,7 +71,7 @@ struct Window { impl Window { /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page /// unless the record crosses it). Unreadable memory: EFAULT. - fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> { let n = count * 4; let cached = self.len != 0 && va >= self.va @@ -108,7 +108,7 @@ struct Checked { /// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is /// spent. -fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { +fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result { if *count >= CDM_MAX_PAGE_CHECKS { return Err(ENOSPC); } @@ -118,7 +118,7 @@ fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. /// The last proven program page is not checked again. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { +fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } @@ -134,7 +134,7 @@ fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Resul /// An indirect launch's 12 argument bytes: readable. The last proven argument page is not /// checked again; arguments that cross a page are checked each time. -fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { +fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result { let end = address.checked_add(12).ok_or(EINVAL)?; let page = address & !(mmu::UAT_PGMSK as u64); let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; @@ -149,7 +149,7 @@ fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { } /// VDM admission: a normal root stream opens with exactly the G15 barrier. -pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { +pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -165,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { /// CDM admission: walks the stream from its base through no-return links until TERMINATE, /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers /// (4), link (8). -pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { +pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result { if !params::stream_gate() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index c9adb8ba8f3910..3a7941b4a36534 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -358,6 +358,20 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +impl VmReadGuard { + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -1804,7 +1818,16 @@ impl Vm { /// mappings cannot change during the copy (GPUVM exec lock). #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index cc636fae0c39c8..1a2e27fd03db4e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -114,6 +114,7 @@ pub const UAT_PGSZ: usize = 0x4000; pub const UAT_PGMSK: usize = UAT_PGSZ - 1; pub struct Vm(pub Vec); + pub type VmReadGuard = Vm; impl Vm { pub fn read_user(&self, va: u64, out: &mut [u8]) -> Result { let start = usize::try_from(va).map_err(|_| EFAULT)?; @@ -133,7 +134,7 @@ /// cover() calls: the page checks the walk made. pub static COVERS: Cell = Cell::new(0); } - pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + pub fn cover(vm: &mut mmu::VmReadGuard, start: u64, end: u64, _: bool, _: bool) -> Result { COVERS.with(|c| c.set(c.get() + 1)); if start >= SYNTHETIC { return Ok(()); @@ -156,34 +157,34 @@ } #[test] fn terminator_and_known_barrier() { - assert_eq!(gate::cdm(&vm(&[0x40000000]), 0, 0), Ok(())); - assert_eq!(gate::cdm(&vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); } #[test] fn cyclic_links_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } /// Words of 4 bytes that fill the byte budget: the root block and one per link. const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] fn byte_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { let mut words = vec![0x60000160; BUDGET_WORDS + 1]; words[BUDGET_WORDS - 1] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); words[BUDGET_WORDS - 1] = 0x60000160; words[BUDGET_WORDS] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); } #[test] fn more_than_8192_records_are_admitted() { // 1024 draws of ten records each, as a tessellated multi-draw writes them. let mut words = vec![0x60000160; 10240]; words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } #[test] fn link_budget_still_bounds_the_walk() { @@ -193,13 +194,13 @@ words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); let mut words = Vec::new(); for i in 0..64u32 { words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } /// The USC window of these cases and its first program, both in the mock's mapped range. const USC: u64 = crate::command::SYNTHETIC; @@ -216,7 +217,7 @@ } fn covers(words: &[u32]) -> (Result, u32) { crate::command::COVERS.with(|c| c.set(0)); - let r = gate::cdm(&vm(words), USC, 0); + let r = gate::cdm(&mut vm(words), USC, 0); (r, crate::command::COVERS.with(|c| c.get())) } const PAGE: u64 = mmu::UAT_PGSZ as u64; @@ -247,9 +248,9 @@ } #[test] fn malformed_and_unreadable_streams_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&mmu::Vm(vec![]), 0, 0), Err(EFAULT)); + assert_eq!(gate::cdm(&mut vm(&[0xffffffff]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut mmu::Vm(vec![]), 0, 0), Err(EFAULT)); } } ''' From 9bb74767dcd4eb3ad62605b5d26b0b758f6b3ca0 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Mon, 26 Oct 2026 03:48:19 +0100 Subject: [PATCH 4/6] soc: apple: report T6030 GPU power to PMP The RTKit AGX vote does not set the PTD power request. PMP receives GPU performance-state updates but ignores its bandwidth demand without that request, leaving memory at its idle floor. Report AGX power after firmware and backend initialization. An earlier report is acknowledged but does not enable bandwidth scaling. Reassert it so a failed early power-off cannot leave a stale report across startup. Use the report driver's lock and checked soc-device table, and clear the report before releasing the vote. Other SoCs keep their existing path. With Mesa unchanged, the packaged kernel takes vkQuake from 104.5 to 372.5 FPS at 1920x1080 and compute-copy bandwidth from 32.7 to 136.3 GB/s. Tested on an M3 Pro J514S. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 8824dd6a7aaa2b302ca7b3a77c2022bacc0fbdc8) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/g15/boot.rs | 1 + drivers/gpu/drm/asahi/g15/glue.c | 6 ++ drivers/gpu/drm/asahi/g15/mod.rs | 16 ++- drivers/pmdomain/apple/pmp-report.c | 56 +++++++++++ drivers/soc/apple/pmp-export.c | 40 +++++++- include/linux/soc/apple/pmp-report.h | 13 +++ include/linux/soc/apple/pmp.h | 6 ++ .../drivers/gpu/asahi-g15/pmp-vote.c | 98 +++++++++++++++++++ .../selftests/drivers/gpu/asahi-g15/run.py | 13 +++ 9 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 38f2444494805c..a58ed36cda60a4 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -73,6 +73,7 @@ pub(crate) mod glue { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index b087bb9ca954ee..b14b30aff3500c 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -35,6 +35,7 @@ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); +int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); @@ -150,6 +151,11 @@ int asahi_g15_pmp_get(struct device *dev) } +int asahi_g15_pmp_ready(void) +{ + return apple_pmp_t6030_gpu_ready(); +} + /* * Give back the vote of asahi_g15_pmp_get() after a start that failed before * the firmware was touched. The full M3 stack's J514S bridge has no release: diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 03eefce9a193cc..ea459e4e4ea5e0 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -539,7 +539,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index d39f6759c5d56c..98d96d08f8a434 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1292,6 +1292,62 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* Find and check the report before taking the GPU vote; optionally update it. */ +static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) +{ + struct apple_pmp_report *rep, *found = NULL; + struct device_node *pmp; + const u8 *table; + int len = 0, ret = -ENODEV; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + + /* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */ + table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len); + if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) || + memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8)) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t6030) + continue; + pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + if (pmp == supplier->of_node) { + of_node_put(pmp); + if (found) + goto out; + found = rep; + } else { + of_node_put(pmp); + } + } + if (found) { + if (readq(found->base + found->offsets->status) != PMP_REPORT_READY) + ret = -EAGAIN; + else + ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0; + } +out: + rcu_read_unlock(); + return ret; +} +int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return apple_pmp_report_t6030_gfx(supplier, NULL); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx); + +/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */ +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return apple_pmp_report_t6030_gfx(supplier, &enable); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 4162d4615d1a63..09804b23b760dc 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) { + bool report_gfx; int ret; if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1) @@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) return ret; - return apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* T6030 needs both the RTKit power vote and the PTD bandwidth report. */ + report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && + command == 0x0f && device_id == 5 && + of_machine_is_compatible("apple,t6030") && + of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2"); + if (report_gfx) { + ret = apple_pmp_report_check_t6030_gfx(pmp_device); + if (ret) + return ret; + } + if (report_gfx && !enabled) { + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) { + /* Clearing was written before the ACK timed out. Restore it. */ + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; + } + } + ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* Power is unknown after a lost reply; assert the report conservatively. */ + if (report_gfx && !enabled && ret) + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; } EXPORT_SYMBOL_GPL(apple_pmp_set_device_power); + +/* Reassert after startup, including a report left set by an earlier failure. */ +int apple_pmp_t6030_gpu_ready(void) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_data) + return -ENODEV; + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) + return ret; + return apple_pmp_report_set_t6030_gfx(pmp_device, true); +} +EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 1bfb4fc69f1f7a..1d0689f31c2039 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -15,6 +15,9 @@ struct device; #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ +int apple_pmp_report_check_t6030_gfx(struct device *supplier); +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); @@ -25,6 +28,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, return -ENODEV; } +static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return -ENODEV; +} + static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 2317ed46f42de1..8b329bf5d4dd5e 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -8,6 +8,8 @@ struct device; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +/* After the acknowledged AGX vote and successful GPU firmware startup. */ +int apple_pmp_t6030_gpu_ready(void); #else static inline int apple_pmp_link_device(struct device *consumer) { @@ -17,6 +19,10 @@ static inline int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enab { return -ENODEV; } +static inline int apple_pmp_t6030_gpu_ready(void) +{ + return -ENODEV; +} #endif /* Registration is private to the PMP Rust driver and its C bridge. */ int apple_pmp_register(struct device *dev, const void *data); diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c new file mode 100644 index 00000000000000..806148f64d1bff --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production PMP vote transaction with failed firmware replies. */ +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define IS_ENABLED(x) (x) +#define guard(kind) ignore_lock +#define dev_warn(...) ((void)0) +static int pmp_lock; +static void ignore_lock(void *p) { (void)p; } +struct device { void *of_node; }; +static struct device owner; +static struct device *pmp_device = &owner; +static const void *pmp_data = &owner; +static bool t6030, requested; +static int ready, check, power_result[2], report_result[3], powers, reports; +static char calls[16]; +static unsigned int used; +static void called(char c) { assert(used < sizeof(calls) - 1); calls[used++] = c; } +static bool of_machine_is_compatible(const char *s) { (void)s; return t6030; } +static bool of_device_is_compatible(void *p, const char *s) +{ + (void)p; (void)s; return t6030; +} +static int apple_pmp_report_wait_supplier_ready(struct device *d, unsigned int ms) +{ + assert(d == &owner && ms == 5000); called('W'); return ready; +} +static int apple_pmp_send_power_command(const void *p, u8 cmd, u16 id, u32 enabled) +{ + assert(p == &owner && cmd == 0x0f && id == 5); + called(enabled ? 'P' : 'p'); assert(powers < 2); return power_result[powers++]; +} +static int apple_pmp_report_check_t6030_gfx(struct device *d) +{ + assert(d == &owner); called('V'); return check; +} +static int apple_pmp_report_set_t6030_gfx(struct device *d, bool enabled) +{ + assert(d == &owner); called(enabled ? 'R' : 'r'); requested = enabled; + assert(reports < 3); return report_result[reports++]; +} +#include "pmp-vote-production.h" +static void reset(void) +{ + pmp_data = &owner; t6030 = true; requested = false; ready = check = 0; powers = reports = used = 0; + memset(calls, 0, sizeof(calls)); + memset(power_result, 0, sizeof(power_result)); + memset(report_result, 0, sizeof(report_result)); +} +static void result(int expected, const char *sequence, bool enabled) +{ + assert(apple_pmp_set_device_power(0x0f, 5, enabled) == expected); + assert(strcmp(calls, sequence) == 0); +} +int main(void) +{ + reset(); + assert(apple_pmp_set_device_power(0, 5, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 16, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 5, 2) == -EINVAL); + assert(!used); + pmp_data = NULL; result(-ENODEV, "", true); + reset(); ready = -EAGAIN; result(-EAGAIN, "W", true); + reset(); t6030 = false; result(0, "WP", true); + reset(); t6030 = false; result(0, "Wp", false); + reset(); check = -EINVAL; result(-EINVAL, "WV", true); + reset(); result(0, "WVP", true); + /* Power-on must not start bandwidth tracking before firmware READY. */ + assert(!reports); + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "WVPrR") == 0 && requested); + reset(); requested = true; + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "rR") == 0 && requested); + reset(); pmp_data = NULL; + assert(apple_pmp_t6030_gpu_ready() == -ENODEV && !used); + reset(); report_result[0] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "r") == 0); + reset(); report_result[1] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "rR") == 0); + reset(); result(0, "WVrp", false); + reset(); power_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVP", true); + reset(); report_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVrR", false); + reset(); report_result[0] = -ETIMEDOUT; report_result[1] = -EIO; + result(-ETIMEDOUT, "WVrR", false); + reset(); power_result[0] = -EIO; result(-EIO, "WVrpR", false); + puts("production PMP vote: scope, sequencing, reply errors and rollback: PASS"); +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 1a2e27fd03db4e..202830cd8d716e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -300,6 +300,19 @@ def main(): str(Path(__file__).with_name("pmp-bridge.c")), "-o", str(output)], check=True) subprocess.run([str(output)], check=True) + owner = (ROOT / "drivers/soc/apple/pmp-export.c").read_text() + start = owner.index("int apple_pmp_set_device_power(") + end = owner.index("\n}", start) + 2 + vote = owner[start:end] + start = owner.index("int apple_pmp_t6030_gpu_ready(") + end = owner.index("\n}", start) + 2 + (temp / "pmp-vote-production.h").write_text(vote + "\n" + owner[start:end]) + output = temp / "pmp-vote" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", + "-Werror", "-I", str(temp), + str(Path(__file__).with_name("pmp-vote.c")), "-o", str(output)], check=True) + subprocess.run([str(output)], check=True) + # Native wire layout is a private per-backend facade. Its independent host test # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. From 9d5fd8f2edca673ce7fa547f99cdc5e9ba914b05 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:28:20 -0700 Subject: [PATCH 5/6] tests/asahi: follow G15 publication binding interfaces Carry the host models forward with the publication-time binding API: provide Job::slot and a separate VM identity for PB growth, and pin the hw.rs golden after proving the only change removes MAX_VMS and its comment. Exercise production bind, release, completion and prefix-admission bodies with explicit modeled slot allocation. Cover retry on slot exhaustion, binding reuse, successful release, failed-job reference lifetime and an append failure after publication, with five additional runtime mutants. Model the T6030 PMP readiness service in the retention harness. Cover readiness failure with and without orphan allocation, successful startup, and exclusion on other platforms, with two additional runtime mutants. Retain the existing failure and removed-fix controls. Clarify last-reference versus fail-stop retention in the ownership comments and format the carried compute match arm. These edits change no production admission, firmware identity, power policy or submission behavior. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/g15/engine.rs | 4 +- drivers/gpu/drm/asahi/g15/job.rs | 2 +- drivers/gpu/drm/asahi/g15/uat.rs | 3 +- drivers/gpu/drm/asahi/g15/vm.rs | 3 +- tools/asahi/g15-input-error-tests.py | 123 ++++++++++++++++++++++++--- tools/asahi/g15-pb-growth-tests.py | 8 ++ tools/asahi/g15-retention-tests.py | 64 +++++++++++--- tools/asahi/g15-zero-words-tests.py | 5 +- 8 files changed, 184 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index ba6408d230da54..80e4a31a78706d 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -920,7 +920,9 @@ impl Firmware { let encoded = (|| { for k in 0..n2 { match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, + Command::Compute(c) => { + self.prefix_compute(&j2, c, &mut pf, f, false)? + } Command::Render(_) => return Err(EINVAL), } } diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index dbca91ee3202b3..ecbe1c8c991aa1 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -21,7 +21,7 @@ //! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage -//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its +//! the firmware may still reference after a fail-stop stays in `Engine::retained` until reboot: its //! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index b37a651689025c..f457b924cbb943 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -26,7 +26,8 @@ //! redundant but harmless.) //! //! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement; failed published jobs retain their slots. +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). //! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 14d98bc4d50b1b..24a912d5968bb6 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -14,7 +14,8 @@ //! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed //! ([`VmState::map_heap`], [`VmState::map_chunk`]); //! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds -//! its binding through completion; failed published jobs retain theirs until reboot. +//! its binding through completion; failed published jobs retain theirs until their last +//! reference drops (until reboot for jobs retained after a fail-stop). //! //! The registry keeps a VM's state while its client handle is open. Queues and jobs retain //! closed VMs until their last reference drops. diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 47971fccfa7106..80103e1bcad0e6 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Actual G15 job dependency completion and G17 status/policy bodies on host services. -DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are -models. No kernel scheduling, GPU access or firmware execution is exercised. +DMA fence lookup, engine submission, queue-fence ordering, VM slot allocation and +kernel mutexes are models. Production job binding/completion and prefix admission +bodies run against them. No kernel scheduling, GPU access or firmware execution +is exercised. """ import argparse import hashlib @@ -33,6 +35,7 @@ def extract(text, marker): type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12); +const ENOSPC:Error=Error(-28);const EAGAIN:Error=Error(-11); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} struct Mirror{word:*mut i32} @@ -44,8 +47,17 @@ def extract(text, marker): mod g17 {use super::*;__POLICY__} mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} -struct Vm{status:VmStatus} -struct Progress{signaled:AtomicBool} +struct Binding{slot:u32,drops:Arc} +impl Binding{fn slot(&self)->u32{self.slot}} +impl Drop for Binding{fn drop(&mut self){self.drops.fetch_add(1,Ordering::SeqCst);}} +struct Vm{status:VmStatus,refuse_binding:AtomicBool,next_slot:AtomicU32,allocations:AtomicU32,binding_drops:Arc} +impl Vm{fn try_bind(&self)->Result{ + if self.refuse_binding.load(Ordering::SeqCst){return Err(ENOSPC);} + self.allocations.fetch_add(1,Ordering::SeqCst); + Ok(Binding{slot:self.next_slot.load(Ordering::SeqCst),drops:self.binding_drops.clone()}) +}} +struct Progress{signaled:AtomicBool,next:AtomicU32,sent:AtomicU32} +impl Progress{fn next(&self)->u32{self.next.load(Ordering::SeqCst)}fn sent(&self)->u32{self.sent.load(Ordering::SeqCst)}} struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} impl JobFence{ @@ -60,12 +72,25 @@ def extract(text, marker): } struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} -struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence,context:AtomicU32,binding:Mutex>} impl Job {__JOB__} +type Completions=Vec; +fn complete(done:&mut Completions,_job:&Arc,result:Result){done.push(result);} +struct Flight{claims:(),group:Vec>} +impl Flight{fn new(_job:Arc)->Self{Self{claims:(),group:vec![]}}} +struct Slots{releases:u32} +impl Slots{fn release(&mut self,_claims:&mut ()){self.releases+=1;}} +struct Engine{slot_wait:bool} +struct HostFirmware{engine:Engine,slots:Slots,publish_result:Result,publishes:u32} +impl HostFirmware{ + fn new(result:Result)->Self{Self{engine:Engine{slot_wait:false},slots:Slots{releases:0},publish_result:result,publishes:0}} + fn publish_prefix_into(&mut self,_job:&Arc,_flight:&mut Flight)->Result{self.publishes+=1;self.publish_result} + __PREFIX__ +} fn make_job(own:Option)->(Arc,Box){ - let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast()),refuse_binding:AtomicBool::new(false),next_slot:AtomicU32::new(7),allocations:AtomicU32::new(0),binding_drops:Arc::new(AtomicU32::new(0))}); if let Some(error)=own{vm.status.record(error);} - let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); (job,mirror) } fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} @@ -89,7 +114,7 @@ def extract(text, marker): assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); // A later job on the same VM is admitted despite the reported client error. - let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); } for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ @@ -110,8 +135,70 @@ def extract(text, marker): } // An engine-owned fault still quarantines before completion. let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + // Actual bind/release bodies: a job keeps its slot until explicitly released. + { + let(job,_mirror)=make_job(None); + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(job.bind(),Err(ENOSPC));assert!(job.binding.lock().is_none()); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"binding must publish its slot"); + job.vm.next_slot.store(9,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"repeated bind must keep existing binding"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),1); + job.release_binding();job.release_binding(); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1,"release_binding must drop exactly once"); + job.bind().unwrap();assert_eq!(job.slot(),9,"released job may acquire a different slot"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),2); + } + // An engine failure must not recycle an ASID while retained job references live. + for result in [Ok(()),Err(EIO),Err(ETIMEDOUT),Err(ECANCELED)]{ + let(job,_mirror)=make_job(None);job.bind().unwrap();let drops=job.vm.binding_drops.clone(); + job.complete(result); + if result.is_err(){ + assert!(job.binding.lock().is_some(),"failed engine completion must retain binding"); + assert_eq!(drops.load(Ordering::SeqCst),0); + job.complete(Ok(()));assert!(job.binding.lock().is_some(),"duplicate completion must retain failed binding"); + let retained=job.clone();drop(job); + assert_eq!(drops.load(Ordering::SeqCst),0,"a retained reference must keep the binding"); + drop(retained);assert_eq!(drops.load(Ordering::SeqCst),1,"last reference releases failed binding"); + }else{ + assert!(job.binding.lock().is_none(),"successful engine completion releases binding"); + assert_eq!(drops.load(Ordering::SeqCst),1);drop(job);assert_eq!(drops.load(Ordering::SeqCst),1); + } + } + // Run the real prefix admission wrapper; only firmware publication is modeled. + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Ok(()));let mut done=vec![]; + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EAGAIN)); + assert!(fw.engine.slot_wait,"slot exhaustion must request retry"); + assert_eq!(fw.publishes,0);assert!(done.is_empty());assert_eq!(job.vm.status.get(),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Ok(()));assert_eq!(fw.publishes,1); + } + for published in [false,true]{ + let(job,_mirror)=make_job(None);job.bind().unwrap(); + job.progress.sent.store(u32::from(published),Ordering::SeqCst); + let mut fw=HostFirmware::new(Err(ENOMEM));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(ENOMEM)); + assert_eq!(fw.slots.releases,1);assert!(done.is_empty()); + if published{ + assert!(job.binding.lock().is_some(),"published prefix must retain binding on append failure"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),0); + }else{ + assert!(job.binding.lock().is_none(),"unpublished failure must release binding"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1); + } + } + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Err(EIO));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EIO)); + assert!(job.binding.lock().is_some(),"firmware-visible failure must retain binding"); + assert_eq!(fw.slots.releases,0); + } assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); - println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); + println!("PASS: production job/prefix/status bodies: input errors, binding reuse/release/retention, exhaustion retry, published-prefix lifetime and isolation"); } ''' @@ -129,16 +216,24 @@ def main(): current = (ROOT / job_path).read_text() status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + engine = (ROOT / "drivers/gpu/drm/asahi/g15/engine.rs").read_text() methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), extract(job, " fn start(self: &Arc")] # The old control needs the current helper for the duplicate-completion test; # it still runs its actual unmodified start, which reproduces the lost input error. methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + # Historical removed-fix controls predate publication bindings. Keep their + # actual start/completion bodies and use current binding helpers alongside. + binding_source = job if "fn release_binding(" in job else current + methods.extend(extract(binding_source, marker) for marker in [ + " pub(crate) fn slot(", " pub(crate) fn bind(", + " pub(crate) fn release_binding("]) bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", " pub(crate) fn reported_error(", " pub(crate) fn report_failure("])).replace( - "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")).replace( + "__PREFIX__", extract(engine, " fn publish_prefix(")) variants = {"control": code} expected = {} if args.mutations: @@ -152,6 +247,11 @@ def main(): "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + "failed-engine-releases-binding": ("if result.is_ok() || !engine {", "if true {", "failed engine completion must retain binding"), + "binding-release-leaks": ("drop(self.binding.lock().take());", "let _ = self;", "release_binding must drop exactly once"), + "bound-job-rebinds": ("if slot.is_none() {", "if true {", "repeated bind must keep existing binding"), + "slot-exhaustion-not-retried": ("self.engine.slot_wait = true;", "self.engine.slot_wait = false;", "slot exhaustion must request retry"), + "published-prefix-releases-binding": ("if job.progress.next() == job.progress.sent() {", "if true {", "published prefix must retain binding on append failure"), } for name, (old, new, message) in changes.items(): if code.count(old) != 1: @@ -160,7 +260,8 @@ def main(): expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, text in [(job_path, job), ("g15/engine.rs", engine), + ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} for name, source in variants.items(): path = args.output / (name + ".rs"); path.write_text(source) binary = args.output / name diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index e7f350d5c7f877..883a8d48a02d53 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -624,6 +624,7 @@ def consts(text, names): pub list: Mutex>, } pub struct VmState { + pub id: u64, pub slot: u32, pub vm: mmu::Vm, pub heap: Arc, @@ -631,6 +632,8 @@ def consts(text, names): impl VmState { pub fn new(slot: u32) -> Arc { Arc::new(VmState { + // Keep VM identity distinct from the modeled hardware slot. + id: 0x10000 + u64::from(slot), slot, vm: mmu::Vm::new(slot), heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), @@ -640,6 +643,11 @@ def consts(text, names): pub struct Job { pub vm: Arc, } + impl Job { + pub fn slot(&self) -> u32 { + self.vm.slot + } + } pub struct Flight { pub job: Job, } diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py index 4812a32b0b1e3e..5a4838ffc6be35 100644 --- a/tools/asahi/g15-retention-tests.py +++ b/tools/asahi/g15-retention-tests.py @@ -34,6 +34,8 @@ def item(text, marker): use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, sync::{Arc, atomic::{AtomicBool, Ordering}}}; type Error = i32; +trait FromErrno { fn from_errno(value: i32) -> Self; } +impl FromErrno for Error { fn from_errno(value: i32) -> Self { value } } type Result = std::result::Result; const GFP_KERNEL: u32 = 0; const EINVAL: Error = -22; @@ -43,6 +45,7 @@ def item(text, marker): static FAIL: Cell = const { Cell::new(0) }; static OWNER_DROPS: Cell = const { Cell::new(0) }; static BUFFER_DROPS: Cell = const { Cell::new(0) }; + static PMP_CALLS: Cell = const { Cell::new(0) }; } fn step(site: u32) -> Result { if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } @@ -72,8 +75,13 @@ def item(text, marker): impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } struct AsahiDevice; impl AsahiDevice { fn as_ref(&self) -> &Self { self } } -struct Config; -static CONFIG: Config = Config; +mod hw { + #[derive(PartialEq)] + pub enum Platform { Adt, AdtWithPmp } +} +struct Config { platform: hw::Platform } +static CONFIG: Config = Config { platform: hw::Platform::Adt }; +static PMP_CONFIG: Config = Config { platform: hw::Platform::AdtWithPmp }; mod mmu { pub struct Uat; } mod regs { pub struct Identity; @@ -152,6 +160,12 @@ def item(text, marker): } mod boot { use super::*; + pub mod glue { + pub unsafe fn asahi_g15_pmp_ready() -> i32 { + super::PMP_CALLS.set(super::PMP_CALLS.get() + 1); + match super::step(13) { Ok(()) => 0, Err(e) => e } + } + } use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } impl Session { @@ -185,7 +199,9 @@ def item(text, marker): let boot::Booted { session, init, objects, channels, events, .. } = booted(); boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } } -fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn reset(fail: u32) { + FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); PMP_CALLS.set(0); +} fn retained(label: &str) { assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); @@ -221,17 +237,32 @@ def item(text, marker): std::mem::forget(shared); } } - reset(0); + for orphan_oom in [false, true] { + reset((1 << 13) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &PMP_CONFIG, booted(), &shared); + assert!(matches!(result, Err(-113)), "PMP readiness error changed"); + assert_eq!(PMP_CALLS.get(), 1, "PMP readiness was not reported"); + retained("PMP readiness failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + for cfg in [&CONFIG, &PMP_CONFIG] { + // A pending PMP error must be irrelevant on a platform without PMP. + reset(if cfg.platform == hw::Platform::Adt { 1 << 13 } else { 0 }); let shared = Shared::new(); let booted = boot::start(&shared).expect("normal boot"); assert_eq!(booted.max_frequency_khz, 1234); let uat: Arc = booted.session.uat(); - let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + let firmware = Firmware::start_backend(&AsahiDevice, cfg, booted, &shared).expect("normal backend"); + assert_eq!(PMP_CALLS.get(), u32::from(cfg.platform == hw::Platform::AdtWithPmp), + "PMP readiness used on the wrong platform"); assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); assert!(!shared.orphaned.load(Ordering::Acquire)); retained("normal success"); std::mem::forget((shared, firmware)); - println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + } + println!("PASS 26 retention cases: orphan OOM, six backend failure sites, four boot-completion sites, PMP readiness failure and both platform success paths"); println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); } ''' @@ -269,11 +300,16 @@ def main(): cwd=ROOT, text=True) variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + variants["ignore-pmp-readiness-error"] = source.replace( + "let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };", + "let _ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; let ret = 0;") + variants["pmp-on-other-platform"] = source.replace( + "if cfg.platform != hw::Platform::AdtWithPmp {", "if false {") receipt = {"scope": __doc__, "sources": { name: hashlib.sha256(text.encode()).hexdigest() for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), - "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + "cases": 26, "public_old_source_control": CONTROL_REF, "variants": {}} for name, text in variants.items(): if name != "actual" and text == source: raise RuntimeError("NOT_APPLIED " + name) @@ -290,14 +326,20 @@ def main(): raise RuntimeError(result.stderr) else: expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", - "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] - if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + "unguarded-registry-allocation": "backend failure: live session owner dropped", + "ignore-pmp-readiness-error": "PMP readiness error changed", + "pmp-on-other-platform": "normal backend: -113"}[name] + if result.returncode == 0 or expected not in result.stderr: raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) - receipt["variants"][name]["intended_owner_drops"] = 5 + if name in ("consume-orphan-before-allocation", "unguarded-registry-allocation"): + if "left: 5" not in result.stderr: + raise RuntimeError("INVALID owner-drop count " + name) + receipt["variants"][name]["intended_owner_drops"] = 5 + receipt["variants"][name]["intended_assertion"] = expected after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, - "checked_cases": 23} + "checked_cases": 26} (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 1439289b1b3247..154c291dda2066 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -11,7 +11,8 @@ CONFIG_DRM_ASAHI_G15 for the T8122 runtime's UAT adoption (two #[cfg] lines; the tables are unchanged), and when the T6031 board source became the device tree with reference words instead of a board file (the Board enum; -the T6030 tables are unchanged). Its T6030 configuration module is +the T6030 tables are unchanged), and for publication-bound slots (removal of +MAX_VMS and its lifetime-slot comment only). Its T6030 configuration module is independently pinned to the original bytes. prepare_board() is pinned too; it was refreshed for the same Board change (its T6030 arm is unchanged). """ @@ -29,7 +30,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "4d34b69f904c6eb864b7d127763480c01a5e1476a8974184364fa4ba3a90ed63", + "hw.rs": "dd024e27c261635c2edd5356f6d79a6e2072ba57f7d13ada0862491b357cdbf3", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From 5af49598aac5ac3d6925b69a900d5417f7e84bd9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:50:29 -0700 Subject: [PATCH 6/6] tests/asahi: model GFX report calls in PMP bridge fixtures The complete tools job found two more host fixtures that predate the T6030 PMP export changes. display26 lacks OF compatibility lookups and GFX-report service definitions; the supplier harness lacks the new report-check and report-update boundaries. Keep display26 on its modeled J613 platform and fail explicitly if it reaches a T6030 report. Model the supplier harness report service under the existing lock, exercise a forwarded report-check error, and require that power-on does not start bandwidth tracking before backend READY. Add a runtime mutant that drops the report-check error. Retain the existing configuration, supplier, readiness and serialization mutants. The production report transaction remains covered separately by the G15 PMP-vote selftest. No production code changes are made here. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- tools/asahi/display26-tests.py | 6 ++++++ tools/asahi/t6030-pmp-supplier-tests.c | 19 +++++++++++++++++++ tools/asahi/t6030-pmp-supplier-tests.py | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 8d2e497fdcf649..2366766042dc2f 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -96,6 +96,7 @@ def main(): #include #include #include +#include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; struct device_link { int unused; }; @@ -117,12 +118,17 @@ def main(): static struct device *get_device(struct device*d){d->refs++;return d;} static void put_device(struct device*d){d->refs--;} static struct device_link *device_link_add(struct device*c,struct device*s,unsigned flags){static struct device_link l;assert(flags==1);return link_fail?NULL:&l;} +/* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ +static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} +static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" static int ready_result,send_result,calls; int apple_pmp_report_wait_supplier_ready(struct device*d,unsigned timeout){assert(d==pmp_device);assert(timeout==5000);return ready_result;} int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} +int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} +int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index bcf94e6244e690..40e6be5bf24c57 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -130,6 +130,8 @@ static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; static int case_count; +/* The report transaction itself is covered by the G15 PMP-vote selftest. */ +static int gfx_check_result, gfx_check_calls, gfx_set_calls; static struct device_node *of_node_get(struct device_node *node) { @@ -375,6 +377,19 @@ static void queue_work(void *wq, int *work) managed_start.result = ready_result; managed_start.started.done = true; } +static int apple_pmp_report_check_t6030_gfx(struct device *dev) +{ + assert(dev == pmp_device && pmp_lock == 1); + gfx_check_calls++; + return gfx_check_result; +} +static int apple_pmp_report_set_t6030_gfx(struct device *dev, bool enabled) +{ + assert(dev == pmp_device && pmp_lock == 1); + (void)enabled; + gfx_set_calls++; + return 0; +} #include "t6030-pmp-production.h" static void balanced(void) @@ -457,6 +472,7 @@ static void reset(void) pmp_data = (void *)1; ready_calls = add_calls = add_result = poll_calls = 0; power_calls = power_result = link_calls = find_calls = 0; + gfx_check_result = gfx_check_calls = gfx_set_calls = 0; lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } @@ -510,6 +526,9 @@ int main(void) balanced(); case_count++; RUN((void)0, 0, 0, 1); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); + RUN(gfx_check_result = -EINVAL, -EINVAL, 0, 0); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); RUN(report14.child = &entry7; entry7.sibling = &entry17; entry17.sibling = NULL, 0, 0, 1); RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index d7934acd3ae98c..57b4900843cf43 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -6,7 +6,9 @@ extracted. OF enumeration/property storage, device binding, MMIO polling, and locks are controlled host boundaries. The T6030 managed entry wait and start wait are extracted too; the firmware/workqueue boundary is controlled. -This models ownership/error handling, not real teardown concurrency or DMA. +The GFX-report service is modeled separately from supplier readiness; its +transaction has its own production-body PMP-vote selftest. This models +ownership/error handling, not real teardown concurrency or DMA. """ import argparse import hashlib @@ -146,6 +148,8 @@ def main(): ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ("gfx-check-error", "ret = apple_pmp_report_check_t6030_gfx(pmp_device);", + "ret = (apple_pmp_report_check_t6030_gfx(pmp_device), 0);"), ] for name, old, new in mutations: assert code.count(old) == 1, (name, code.count(old))